rpc-send-tx-svc server-side retry knobs (backport #20818) (#20830)

* rpc-send-tx-svc: add with_config constructor

(cherry picked from commit fe098b5ddc)

# Conflicts:
#	Cargo.lock
#	core/Cargo.toml
#	replica-node/Cargo.toml
#	rpc/src/rpc_service.rs
#	rpc/src/send_transaction_service.rs
#	validator/Cargo.toml

* rpc-send-tx-svc: server-side retry knobs

(cherry picked from commit 2744a2128c)

Co-authored-by: Trent Nelson <trent@solana.com>

.travis.yml

@@ -123,7 +123,7 @@ jobs:
       if: type IN (push, pull_request) OR tag IS present
       language: node_js
       node_js:
-        - "node"
+        - "lts/*"
 
       services:
         - docker

Cargo.toml

@@ -1,5 +1,8 @@
 [workspace]
 members = [
+    "accountsdb-plugin-interface",
+    "accountsdb-plugin-manager",
+    "accountsdb-plugin-postgres",
     "accounts-cluster-bench",
     "bench-exchange",
     "bench-streamer",
@@ -43,11 +46,10 @@ members = [
     "poh-bench",
     "program-test",
     "programs/bpf_loader",
+    "programs/compute-budget",
     "programs/config",
     "programs/exchange",
-    "programs/failure",
-    "programs/noop",
-    "programs/ownable",
+    "programs/ed25519",
     "programs/secp256k1",
     "programs/stake",
     "programs/vote",

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.7.15"
+version = "1.8.1"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,9 +19,9 @@ lazy_static = "1.4.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-config-program = { path = "../programs/config", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.15" }
+solana-config-program = { path = "../programs/config", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.8.1" }
 spl-token-v2-0 = { package = "spl-token", version = "=3.2.0", features = ["no-entrypoint"] }
 thiserror = "1.0"
 zstd = "0.5.1"

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.7.15"
+version = "1.8.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,11 +11,11 @@ publish = false
 [dependencies]
 log = "0.4.11"
 rayon = "1.5.0"
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
-solana-measure = { path = "../measure", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
+solana-measure = { path = "../measure", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

accounts-bench/src/main.rs

@@ -66,6 +66,7 @@ fn main() {
         AccountSecondaryIndexes::default(),
         false,
         AccountShrinkThreshold::default(),
+        None,
     );
     println!("Creating {} accounts", num_accounts);
     let mut create_time = Measure::start("create accounts");

accounts-cluster-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-cluster-bench"
-version = "1.7.15"
+version = "1.8.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,24 +13,24 @@ clap = "2.33.1"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.4.1"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.15" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-client = { path = "../client", version = "=1.7.15" }
-solana-core = { path = "../core", version = "=1.7.15" }
-solana-faucet = { path = "../faucet", version = "=1.7.15" }
-solana-gossip = { path = "../gossip", version = "=1.7.15" }
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-measure = { path = "../measure", version = "=1.7.15" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.15" }
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-streamer = { path = "../streamer", version = "=1.7.15" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.8.1" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-client = { path = "../client", version = "=1.8.1" }
+solana-core = { path = "../core", version = "=1.8.1" }
+solana-faucet = { path = "../faucet", version = "=1.8.1" }
+solana-gossip = { path = "../gossip", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-measure = { path = "../measure", version = "=1.8.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-streamer = { path = "../streamer", version = "=1.8.1" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
 spl-token-v2-0 = { package = "spl-token", version = "=3.2.0", features = ["no-entrypoint"] }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "=1.7.15" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.8.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-interface/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2018"
+name = "solana-accountsdb-plugin-interface"
+description = "The Solana AccountsDb plugin interface."
+version = "1.8.1"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-validator"
+
+[dependencies]
+log = "0.4.11"
+thiserror = "1.0.29"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-interface/README.md

@@ -0,0 +1,20 @@
+<p align="center">
+  <a href="https://solana.com">
+    <img alt="Solana" src="https://i.imgur.com/IKyzQ6T.png" width="250" />
+  </a>
+</p>
+
+# Solana AccountsDb Plugin Interface
+
+This crate enables an AccountsDb plugin to be plugged into the Solana Validator runtime to take actions
+at the time of each account update; for example, saving the account state to an external database. The plugin must implement the `AccountsDbPlugin` trait. Please see the detail of the `accountsdb_plugin_interface.rs` for the interface definition.
+
+The plugin should produce a `cdylib` dynamic library, which must expose a `C` function `_create_plugin()` that
+instantiates the implementation of the interface.
+
+The `solana-accountsdb-plugin-postgres` crate provides an example of how to create a plugin which saves the accounts data into an
+external PostgreSQL databases.
+
+More information about Solana is available in the [Solana documentation](https://docs.solana.com/).
+
+Still have questions?  Ask us on [Discord](https://discordapp.com/invite/pquxPsq)

accountsdb-plugin-interface/src/accountsdb_plugin_interface.rs

@@ -0,0 +1,90 @@
+/// The interface for AccountsDb plugins. A plugin must implement
+/// the AccountsDbPlugin trait to work with the runtime.
+/// In addition, the dynamic library must export a "C" function _create_plugin which
+/// creates the implementation of the plugin.
+use {
+    std::{any::Any, error, io},
+    thiserror::Error,
+};
+
+impl Eq for ReplicaAccountInfo<'_> {}
+
+#[derive(Clone, PartialEq, Debug)]
+pub struct ReplicaAccountInfo<'a> {
+    pub pubkey: &'a [u8],
+    pub lamports: u64,
+    pub owner: &'a [u8],
+    pub executable: bool,
+    pub rent_epoch: u64,
+    pub data: &'a [u8],
+}
+
+pub enum ReplicaAccountInfoVersions<'a> {
+    V0_0_1(&'a ReplicaAccountInfo<'a>),
+}
+
+#[derive(Error, Debug)]
+pub enum AccountsDbPluginError {
+    #[error("Error opening config file.")]
+    ConfigFileOpenError(#[from] io::Error),
+
+    #[error("Error reading config file.")]
+    ConfigFileReadError { msg: String },
+
+    #[error("Error updating account.")]
+    AccountsUpdateError { msg: String },
+
+    #[error("Error updating slot status.")]
+    SlotStatusUpdateError { msg: String },
+
+    #[error("Plugin-defined custom error.")]
+    Custom(Box<dyn error::Error + Send + Sync>),
+}
+
+#[derive(Debug, Clone)]
+pub enum SlotStatus {
+    Processed,
+    Rooted,
+    Confirmed,
+}
+
+impl SlotStatus {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            SlotStatus::Confirmed => "confirmed",
+            SlotStatus::Processed => "processed",
+            SlotStatus::Rooted => "rooted",
+        }
+    }
+}
+
+pub type Result<T> = std::result::Result<T, AccountsDbPluginError>;
+
+pub trait AccountsDbPlugin: Any + Send + Sync + std::fmt::Debug {
+    fn name(&self) -> &'static str;
+
+    /// The callback called when a plugin is loaded by the system,
+    /// used for doing whatever initialization is required by the plugin.
+    /// The _config_file contains the name of the
+    /// of the config file. The config must be in JSON format and
+    /// include a field "libpath" indicating the full path
+    /// name of the shared library implementing this interface.
+    fn on_load(&mut self, _config_file: &str) -> Result<()> {
+        Ok(())
+    }
+
+    /// The callback called right before a plugin is unloaded by the system
+    /// Used for doing cleanup before unload.
+    fn on_unload(&mut self) {}
+
+    /// Called when an account is updated at a slot.
+    fn update_account(&mut self, account: ReplicaAccountInfoVersions, slot: u64) -> Result<()>;
+
+    /// Called when a slot status is updated
+    fn update_slot_status(
+        &mut self,
+        slot: u64,
+        parent: Option<u64>,
+        status: SlotStatus,
+    ) -> Result<()>;
+}

accountsdb-plugin-interface/src/lib.rs

@@ -0,0 +1 @@
+pub mod accountsdb_plugin_interface;

accountsdb-plugin-manager/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2018"
+name = "solana-accountsdb-plugin-manager"
+description = "The Solana AccountsDb plugin manager."
+version = "1.8.1"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-validator"
+
+[dependencies]
+bs58 = "0.4.0"
+crossbeam-channel = "0.4"
+libloading = "0.7.0"
+log = "0.4.11"
+serde = "1.0.130"
+serde_derive = "1.0.103"
+serde_json = "1.0.67"
+solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-measure = { path = "../measure", version = "=1.8.1" }
+solana-metrics = { path = "../metrics", version = "=1.8.1" }
+solana-rpc = { path = "../rpc", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+thiserror = "1.0.21"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-manager/src/accounts_update_notifier.rs

@@ -0,0 +1,157 @@
+/// Module responsible for notifying plugins of account updates
+use {
+    crate::accountsdb_plugin_manager::AccountsDbPluginManager,
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        ReplicaAccountInfo, ReplicaAccountInfoVersions, SlotStatus,
+    },
+    solana_measure::measure::Measure,
+    solana_metrics::*,
+    solana_runtime::{
+        accounts_update_notifier_interface::AccountsUpdateNotifierInterface,
+        append_vec::StoredAccountMeta,
+    },
+    solana_sdk::{
+        account::{AccountSharedData, ReadableAccount},
+        clock::Slot,
+        pubkey::Pubkey,
+    },
+    std::sync::{Arc, RwLock},
+};
+#[derive(Debug)]
+pub(crate) struct AccountsUpdateNotifierImpl {
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+}
+
+impl AccountsUpdateNotifierInterface for AccountsUpdateNotifierImpl {
+    fn notify_account_update(&self, slot: Slot, pubkey: &Pubkey, account: &AccountSharedData) {
+        if let Some(account_info) = self.accountinfo_from_shared_account_data(pubkey, account) {
+            self.notify_plugins_of_account_update(account_info, slot);
+        }
+    }
+
+    fn notify_account_restore_from_snapshot(&self, slot: Slot, account: &StoredAccountMeta) {
+        if let Some(account_info) = self.accountinfo_from_stored_account_meta(account) {
+            self.notify_plugins_of_account_update(account_info, slot);
+        }
+    }
+
+    fn notify_slot_confirmed(&self, slot: Slot, parent: Option<Slot>) {
+        self.notify_slot_status(slot, parent, SlotStatus::Confirmed);
+    }
+
+    fn notify_slot_processed(&self, slot: Slot, parent: Option<Slot>) {
+        self.notify_slot_status(slot, parent, SlotStatus::Processed);
+    }
+
+    fn notify_slot_rooted(&self, slot: Slot, parent: Option<Slot>) {
+        self.notify_slot_status(slot, parent, SlotStatus::Rooted);
+    }
+}
+
+impl AccountsUpdateNotifierImpl {
+    pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self {
+        AccountsUpdateNotifierImpl { plugin_manager }
+    }
+
+    fn accountinfo_from_shared_account_data<'a>(
+        &self,
+        pubkey: &'a Pubkey,
+        account: &'a AccountSharedData,
+    ) -> Option<ReplicaAccountInfo<'a>> {
+        Some(ReplicaAccountInfo {
+            pubkey: pubkey.as_ref(),
+            lamports: account.lamports(),
+            owner: account.owner().as_ref(),
+            executable: account.executable(),
+            rent_epoch: account.rent_epoch(),
+            data: account.data(),
+        })
+    }
+
+    fn accountinfo_from_stored_account_meta<'a>(
+        &self,
+        stored_account_meta: &'a StoredAccountMeta,
+    ) -> Option<ReplicaAccountInfo<'a>> {
+        Some(ReplicaAccountInfo {
+            pubkey: stored_account_meta.meta.pubkey.as_ref(),
+            lamports: stored_account_meta.account_meta.lamports,
+            owner: stored_account_meta.account_meta.owner.as_ref(),
+            executable: stored_account_meta.account_meta.executable,
+            rent_epoch: stored_account_meta.account_meta.rent_epoch,
+            data: stored_account_meta.data,
+        })
+    }
+
+    fn notify_plugins_of_account_update(&self, account: ReplicaAccountInfo, slot: Slot) {
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+        for plugin in plugin_manager.plugins.iter_mut() {
+            let mut measure = Measure::start("accountsdb-plugin-update-account");
+            match plugin.update_account(ReplicaAccountInfoVersions::V0_0_1(&account), slot) {
+                Err(err) => {
+                    error!(
+                        "Failed to update account {} at slot {}, error: {} to plugin {}",
+                        bs58::encode(account.pubkey).into_string(),
+                        slot,
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully updated account {} at slot {} to plugin {}",
+                        bs58::encode(account.pubkey).into_string(),
+                        slot,
+                        plugin.name()
+                    );
+                }
+            }
+            measure.stop();
+            inc_new_counter_info!(
+                "accountsdb-plugin-update-account-ms",
+                measure.as_ms() as usize,
+                100000,
+                100000
+            );
+        }
+    }
+
+    pub fn notify_slot_status(&self, slot: Slot, parent: Option<Slot>, slot_status: SlotStatus) {
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+
+        for plugin in plugin_manager.plugins.iter_mut() {
+            let mut measure = Measure::start("accountsdb-plugin-update-slot");
+            match plugin.update_slot_status(slot, parent, slot_status.clone()) {
+                Err(err) => {
+                    error!(
+                        "Failed to update slot status at slot {}, error: {} to plugin {}",
+                        slot,
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully updated slot status at slot {} to plugin {}",
+                        slot,
+                        plugin.name()
+                    );
+                }
+            }
+            measure.stop();
+            inc_new_counter_info!(
+                "accountsdb-plugin-update-slot-ms",
+                measure.as_ms() as usize,
+                1000,
+                1000
+            );
+        }
+    }
+}

accountsdb-plugin-manager/src/accountsdb_plugin_manager.rs

@@ -0,0 +1,55 @@
+/// Managing the AccountsDb plugins
+use {
+    libloading::{Library, Symbol},
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::AccountsDbPlugin,
+    std::error::Error,
+};
+
+#[derive(Default, Debug)]
+pub struct AccountsDbPluginManager {
+    pub plugins: Vec<Box<dyn AccountsDbPlugin>>,
+    libs: Vec<Library>,
+}
+
+impl AccountsDbPluginManager {
+    pub fn new() -> Self {
+        AccountsDbPluginManager {
+            plugins: Vec::default(),
+            libs: Vec::default(),
+        }
+    }
+
+    /// # Safety
+    ///
+    /// This function loads the dynamically linked library specified in the path. The library
+    /// must do necessary initializations.
+    pub unsafe fn load_plugin(
+        &mut self,
+        libpath: &str,
+        config_file: &str,
+    ) -> Result<(), Box<dyn Error>> {
+        type PluginConstructor = unsafe fn() -> *mut dyn AccountsDbPlugin;
+        let lib = Library::new(libpath)?;
+        let constructor: Symbol<PluginConstructor> = lib.get(b"_create_plugin")?;
+        let plugin_raw = constructor();
+        let mut plugin = Box::from_raw(plugin_raw);
+        plugin.on_load(config_file)?;
+        self.plugins.push(plugin);
+        self.libs.push(lib);
+        Ok(())
+    }
+
+    /// Unload all plugins and loaded plugin libraries, making sure to fire
+    /// their `on_plugin_unload()` methods so they can do any necessary cleanup.
+    pub fn unload(&mut self) {
+        for mut plugin in self.plugins.drain(..) {
+            info!("Unloading plugin for {:?}", plugin.name());
+            plugin.on_unload();
+        }
+
+        for lib in self.libs.drain(..) {
+            drop(lib);
+        }
+    }
+}

accountsdb-plugin-manager/src/accountsdb_plugin_service.rs

@@ -0,0 +1,157 @@
+use {
+    crate::{
+        accounts_update_notifier::AccountsUpdateNotifierImpl,
+        accountsdb_plugin_manager::AccountsDbPluginManager,
+        slot_status_observer::SlotStatusObserver,
+    },
+    crossbeam_channel::Receiver,
+    log::*,
+    serde_json,
+    solana_rpc::optimistically_confirmed_bank_tracker::BankNotification,
+    solana_runtime::accounts_update_notifier_interface::AccountsUpdateNotifier,
+    std::{
+        fs::File,
+        io::Read,
+        path::{Path, PathBuf},
+        sync::{Arc, RwLock},
+        thread,
+    },
+    thiserror::Error,
+};
+
+#[derive(Error, Debug)]
+pub enum AccountsdbPluginServiceError {
+    #[error("Cannot open the the plugin config file")]
+    CannotOpenConfigFile(String),
+
+    #[error("Cannot read the the plugin config file")]
+    CannotReadConfigFile(String),
+
+    #[error("The config file is not in a valid Json format")]
+    InvalidConfigFileFormat(String),
+
+    #[error("Plugin library path is not specified in the config file")]
+    LibPathNotSet,
+
+    #[error("Invalid plugin path")]
+    InvalidPluginPath,
+
+    #[error("Cannot load plugin shared library")]
+    PluginLoadError(String),
+}
+
+/// The service managing the AccountsDb plugin workflow.
+pub struct AccountsDbPluginService {
+    slot_status_observer: SlotStatusObserver,
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+    accounts_update_notifier: AccountsUpdateNotifier,
+}
+
+impl AccountsDbPluginService {
+    /// Creates and returns the AccountsDbPluginService.
+    /// # Arguments
+    /// * `confirmed_bank_receiver` - The receiver for confirmed bank notification
+    /// * `accountsdb_plugin_config_file` - The config file path for the plugin. The
+    ///    config file controls the plugin responsible
+    ///    for transporting the data to external data stores. It is defined in JSON format.
+    ///    The `libpath` field should be pointed to the full path of the dynamic shared library
+    ///    (.so file) to be loaded. The shared library must implement the `AccountsDbPlugin`
+    ///    trait. And the shared library shall export a `C` function `_create_plugin` which
+    ///    shall create the implementation of `AccountsDbPlugin` and returns to the caller.
+    ///    The rest of the JSON fields' definition is up to to the concrete plugin implementation
+    ///    It is usually used to configure the connection information for the external data store.
+
+    pub fn new(
+        confirmed_bank_receiver: Receiver<BankNotification>,
+        accountsdb_plugin_config_files: &[PathBuf],
+    ) -> Result<Self, AccountsdbPluginServiceError> {
+        info!(
+            "Starting AccountsDbPluginService from config files: {:?}",
+            accountsdb_plugin_config_files
+        );
+        let mut plugin_manager = AccountsDbPluginManager::new();
+
+        for accountsdb_plugin_config_file in accountsdb_plugin_config_files {
+            Self::load_plugin(&mut plugin_manager, accountsdb_plugin_config_file)?;
+        }
+
+        let plugin_manager = Arc::new(RwLock::new(plugin_manager));
+        let accounts_update_notifier = Arc::new(RwLock::new(AccountsUpdateNotifierImpl::new(
+            plugin_manager.clone(),
+        )));
+        let slot_status_observer =
+            SlotStatusObserver::new(confirmed_bank_receiver, accounts_update_notifier.clone());
+
+        info!("Started AccountsDbPluginService");
+        Ok(AccountsDbPluginService {
+            slot_status_observer,
+            plugin_manager,
+            accounts_update_notifier,
+        })
+    }
+
+    fn load_plugin(
+        plugin_manager: &mut AccountsDbPluginManager,
+        accountsdb_plugin_config_file: &Path,
+    ) -> Result<(), AccountsdbPluginServiceError> {
+        let mut file = match File::open(accountsdb_plugin_config_file) {
+            Ok(file) => file,
+            Err(err) => {
+                return Err(AccountsdbPluginServiceError::CannotOpenConfigFile(format!(
+                    "Failed to open the plugin config file {:?}, error: {:?}",
+                    accountsdb_plugin_config_file, err
+                )));
+            }
+        };
+
+        let mut contents = String::new();
+        if let Err(err) = file.read_to_string(&mut contents) {
+            return Err(AccountsdbPluginServiceError::CannotReadConfigFile(format!(
+                "Failed to read the plugin config file {:?}, error: {:?}",
+                accountsdb_plugin_config_file, err
+            )));
+        }
+
+        let result: serde_json::Value = match serde_json::from_str(&contents) {
+            Ok(value) => value,
+            Err(err) => {
+                return Err(AccountsdbPluginServiceError::InvalidConfigFileFormat(
+                    format!(
+                        "The config file {:?} is not in a valid Json format, error: {:?}",
+                        accountsdb_plugin_config_file, err
+                    ),
+                ));
+            }
+        };
+
+        let libpath = result["libpath"]
+            .as_str()
+            .ok_or(AccountsdbPluginServiceError::LibPathNotSet)?;
+        let config_file = accountsdb_plugin_config_file
+            .as_os_str()
+            .to_str()
+            .ok_or(AccountsdbPluginServiceError::InvalidPluginPath)?;
+
+        unsafe {
+            let result = plugin_manager.load_plugin(libpath, config_file);
+            if let Err(err) = result {
+                let msg = format!(
+                    "Failed to load the plugin library: {:?}, error: {:?}",
+                    libpath, err
+                );
+                return Err(AccountsdbPluginServiceError::PluginLoadError(msg));
+            }
+        }
+        Ok(())
+    }
+
+    pub fn get_accounts_update_notifier(&self) -> AccountsUpdateNotifier {
+        self.accounts_update_notifier.clone()
+    }
+
+    pub fn join(mut self) -> thread::Result<()> {
+        self.slot_status_observer.join()?;
+        self.plugin_manager.write().unwrap().unload();
+        Ok(())
+    }
+}

accountsdb-plugin-manager/src/lib.rs

@@ -0,0 +1,4 @@
+pub mod accounts_update_notifier;
+pub mod accountsdb_plugin_manager;
+pub mod accountsdb_plugin_service;
+pub mod slot_status_observer;

accountsdb-plugin-manager/src/slot_status_observer.rs

@@ -0,0 +1,80 @@
+use {
+    crossbeam_channel::Receiver,
+    solana_rpc::optimistically_confirmed_bank_tracker::BankNotification,
+    solana_runtime::accounts_update_notifier_interface::AccountsUpdateNotifier,
+    std::{
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc,
+        },
+        thread::{self, Builder, JoinHandle},
+    },
+};
+
+#[derive(Debug)]
+pub(crate) struct SlotStatusObserver {
+    bank_notification_receiver_service: Option<JoinHandle<()>>,
+    exit_updated_slot_server: Arc<AtomicBool>,
+}
+
+impl SlotStatusObserver {
+    pub fn new(
+        bank_notification_receiver: Receiver<BankNotification>,
+        accounts_update_notifier: AccountsUpdateNotifier,
+    ) -> Self {
+        let exit_updated_slot_server = Arc::new(AtomicBool::new(false));
+
+        Self {
+            bank_notification_receiver_service: Some(Self::run_bank_notification_receiver(
+                bank_notification_receiver,
+                exit_updated_slot_server.clone(),
+                accounts_update_notifier,
+            )),
+            exit_updated_slot_server,
+        }
+    }
+
+    pub fn join(&mut self) -> thread::Result<()> {
+        self.exit_updated_slot_server.store(true, Ordering::Relaxed);
+        self.bank_notification_receiver_service
+            .take()
+            .map(JoinHandle::join)
+            .unwrap()
+    }
+
+    fn run_bank_notification_receiver(
+        bank_notification_receiver: Receiver<BankNotification>,
+        exit: Arc<AtomicBool>,
+        accounts_update_notifier: AccountsUpdateNotifier,
+    ) -> JoinHandle<()> {
+        Builder::new()
+            .name("bank_notification_receiver".to_string())
+            .spawn(move || {
+                while !exit.load(Ordering::Relaxed) {
+                    if let Ok(slot) = bank_notification_receiver.recv() {
+                        match slot {
+                            BankNotification::OptimisticallyConfirmed(slot) => {
+                                accounts_update_notifier
+                                    .read()
+                                    .unwrap()
+                                    .notify_slot_confirmed(slot, None);
+                            }
+                            BankNotification::Frozen(bank) => {
+                                accounts_update_notifier
+                                    .read()
+                                    .unwrap()
+                                    .notify_slot_processed(bank.slot(), Some(bank.parent_slot()));
+                            }
+                            BankNotification::Root(bank) => {
+                                accounts_update_notifier
+                                    .read()
+                                    .unwrap()
+                                    .notify_slot_rooted(bank.slot(), Some(bank.parent_slot()));
+                            }
+                        }
+                    }
+                }
+            })
+            .unwrap()
+    }
+}

accountsdb-plugin-postgres/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2018"
+name = "solana-accountsdb-plugin-postgres"
+description = "The Solana AccountsDb plugin for PostgreSQL database."
+version = "1.8.1"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-validator"
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+bs58 = "0.4.0"
+chrono = { version = "0.4.11", features = ["serde"] }
+crossbeam-channel = "0.5"
+log = "0.4.14"
+postgres = { version = "0.19.1", features = ["with-chrono-0_4"] }
+serde = "1.0.130"
+serde_derive = "1.0.103"
+serde_json = "1.0.67"
+solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-metrics = { path = "../metrics", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+thiserror = "1.0.21"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-postgres/README.md

@@ -0,0 +1,5 @@
+This is an example implementing the AccountsDb plugin for PostgreSQL database.
+Please see the `src/accountsdb_plugin_postgres.rs` for the format of the plugin's configuration file.
+
+To create the schema objects for the database, please use `scripts/create_schema.sql`.
+`scripts/drop_schema.sql` can be used to tear down the schema objects.

accountsdb-plugin-postgres/scripts/create_schema.sql

@@ -0,0 +1,52 @@
+/**
+ * This plugin implementation for PostgreSQL requires the following tables
+ */
+-- The table storing accounts
+
+
+CREATE TABLE account (
+    pubkey BYTEA PRIMARY KEY,
+    owner BYTEA,
+    lamports BIGINT NOT NULL,
+    slot BIGINT NOT NULL,
+    executable BOOL NOT NULL,
+    rent_epoch BIGINT NOT NULL,
+    data BYTEA,
+    updated_on TIMESTAMP NOT NULL
+);
+
+-- The table storing slot information
+CREATE TABLE slot (
+    slot BIGINT PRIMARY KEY,
+    parent BIGINT,
+    status varchar(16) NOT NULL,
+    updated_on TIMESTAMP NOT NULL
+);
+
+/**
+ * The following is for keeping historical data for accounts and is not required for plugin to work.
+ */
+-- The table storing historical data for accounts
+CREATE TABLE account_audit (
+    pubkey BYTEA,
+    owner BYTEA,
+    lamports BIGINT NOT NULL,
+    slot BIGINT NOT NULL,
+    executable BOOL NOT NULL,
+    rent_epoch BIGINT NOT NULL,
+    data BYTEA,
+    updated_on TIMESTAMP NOT NULL
+);
+
+CREATE FUNCTION audit_account_update() RETURNS trigger AS $audit_account_update$
+    BEGIN
+		INSERT INTO account_audit (pubkey, owner, lamports, slot, executable, rent_epoch, data, updated_on)
+            VALUES (OLD.pubkey, OLD.owner, OLD.lamports, OLD.slot,
+                    OLD.executable, OLD.rent_epoch, OLD.data, OLD.updated_on);
+        RETURN NEW;
+    END;
+
+$audit_account_update$ LANGUAGE plpgsql;
+
+CREATE TRIGGER account_update_trigger AFTER UPDATE OR DELETE ON account
+    FOR EACH ROW EXECUTE PROCEDURE audit_account_update();

accountsdb-plugin-postgres/scripts/drop_schema.sql

@@ -0,0 +1,9 @@
+/**
+ * Script for cleaning up the schema for PostgreSQL used for the AccountsDb plugin.
+ */
+
+DROP TRIGGER account_update_trigger;
+DROP FUNCTION audit_account_update;
+DROP TABLE account_audit;
+DROP TABLE account;
+DROP TABLE slot;

accountsdb-plugin-postgres/src/accounts_selector.rs

@@ -0,0 +1,69 @@
+use {log::*, std::collections::HashSet};
+
+#[derive(Debug)]
+pub(crate) struct AccountsSelector {
+    pub accounts: HashSet<Vec<u8>>,
+    pub owners: HashSet<Vec<u8>>,
+    pub select_all_accounts: bool,
+}
+
+impl AccountsSelector {
+    pub fn default() -> Self {
+        AccountsSelector {
+            accounts: HashSet::default(),
+            owners: HashSet::default(),
+            select_all_accounts: true,
+        }
+    }
+
+    pub fn new(accounts: &[String], owners: &[String]) -> Self {
+        info!(
+            "Creating AccountsSelector from accounts: {:?}, owners: {:?}",
+            accounts, owners
+        );
+
+        let select_all_accounts = accounts.iter().any(|key| key == "*");
+        if select_all_accounts {
+            return AccountsSelector {
+                accounts: HashSet::default(),
+                owners: HashSet::default(),
+                select_all_accounts,
+            };
+        }
+        let accounts = accounts
+            .iter()
+            .map(|key| bs58::decode(key).into_vec().unwrap())
+            .collect();
+        let owners = owners
+            .iter()
+            .map(|key| bs58::decode(key).into_vec().unwrap())
+            .collect();
+        AccountsSelector {
+            accounts,
+            owners,
+            select_all_accounts,
+        }
+    }
+
+    pub fn is_account_selected(&self, account: &[u8], owner: &[u8]) -> bool {
+        self.select_all_accounts || self.accounts.contains(account) || self.owners.contains(owner)
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::*;
+
+    #[test]
+    fn test_create_accounts_selector() {
+        AccountsSelector::new(
+            &["9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin".to_string()],
+            &[],
+        );
+
+        AccountsSelector::new(
+            &[],
+            &["9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin".to_string()],
+        );
+    }
+}

accountsdb-plugin-postgres/src/accountsdb_plugin_postgres.rs

@@ -0,0 +1,297 @@
+/// Main entry for the PostgreSQL plugin
+use {
+    crate::{
+        accounts_selector::AccountsSelector,
+        postgres_client::{
+            ParallelPostgresClient, PostgresClient, PostgresClientBuilder, SimplePostgresClient,
+        },
+    },
+    bs58,
+    log::*,
+    serde_derive::{Deserialize, Serialize},
+    serde_json,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        AccountsDbPlugin, AccountsDbPluginError, ReplicaAccountInfoVersions, Result, SlotStatus,
+    },
+    std::{fs::File, io::Read},
+    thiserror::Error,
+};
+
+enum PostgresClientEnum {
+    Simple(SimplePostgresClient),
+    Parallel(ParallelPostgresClient),
+}
+
+#[derive(Default)]
+pub struct AccountsDbPluginPostgres {
+    client: Option<PostgresClientEnum>,
+    accounts_selector: Option<AccountsSelector>,
+}
+
+impl std::fmt::Debug for AccountsDbPluginPostgres {
+    fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct AccountsDbPluginPostgresConfig {
+    pub host: String,
+    pub user: String,
+    pub threads: Option<usize>,
+}
+
+#[derive(Error, Debug)]
+pub enum AccountsDbPluginPostgresError {
+    #[error("Error connecting to the backend data store.")]
+    DataStoreConnectionError { msg: String },
+
+    #[error("Error preparing data store schema.")]
+    DataSchemaError { msg: String },
+}
+
+impl AccountsDbPlugin for AccountsDbPluginPostgres {
+    fn name(&self) -> &'static str {
+        "AccountsDbPluginPostgres"
+    }
+
+    /// Do initialization for the PostgreSQL plugin.
+    /// # Arguments
+    ///
+    /// Format of the config file:
+    /// The `accounts_selector` section allows the user to controls accounts selections.
+    /// "accounts_selector" : {
+    ///     "accounts" : \["pubkey-1", "pubkey-2", ..., "pubkey-n"\],
+    /// }
+    /// or:
+    /// "accounts_selector" = {
+    ///     "owners" : \["pubkey-1", "pubkey-2", ..., "pubkey-m"\]
+    /// }
+    /// Accounts either satisyfing the accounts condition or owners condition will be selected.
+    /// When only owners is specified,
+    /// all accounts belonging to the owners will be streamed.
+    /// The accounts field support wildcard to select all accounts:
+    /// "accounts_selector" : {
+    ///     "accounts" : \["*"\],
+    /// }
+    /// "host" specifies the PostgreSQL server.
+    /// "user" specifies the PostgreSQL user.
+    /// "threads" optional, specifies the number of worker threads for the plugin. A thread
+    /// maintains a PostgreSQL connection to the server.
+    /// # Examples
+    /// {
+    ///    "libpath": "/home/solana/target/release/libsolana_accountsdb_plugin_postgres.so",
+    ///    "host": "host_foo",
+    ///    "user": "solana",
+    ///    "threads": 10,
+    ///    "accounts_selector" : {
+    ///       "owners" : ["9oT9R5ZyRovSVnt37QvVoBttGpNqR3J7unkb567NP8k3"]
+    /// }
+
+    fn on_load(&mut self, config_file: &str) -> Result<()> {
+        solana_logger::setup_with_default("info");
+        info!(
+            "Loading plugin {:?} from config_file {:?}",
+            self.name(),
+            config_file
+        );
+        let mut file = File::open(config_file)?;
+        let mut contents = String::new();
+        file.read_to_string(&mut contents)?;
+
+        let result: serde_json::Value = serde_json::from_str(&contents).unwrap();
+        self.accounts_selector = Some(Self::create_accounts_selector_from_config(&result));
+
+        let result: serde_json::Result<AccountsDbPluginPostgresConfig> =
+            serde_json::from_str(&contents);
+        match result {
+            Err(err) => {
+                return Err(AccountsDbPluginError::ConfigFileReadError {
+                    msg: format!(
+                        "The config file is not in the JSON format expected: {:?}",
+                        err
+                    ),
+                })
+            }
+            Ok(config) => {
+                self.client = if config.threads.is_some() && config.threads.unwrap() > 1 {
+                    let client = PostgresClientBuilder::build_pararallel_postgres_client(&config)?;
+                    Some(PostgresClientEnum::Parallel(client))
+                } else {
+                    let client = PostgresClientBuilder::build_simple_postgres_client(&config)?;
+                    Some(PostgresClientEnum::Simple(client))
+                };
+            }
+        }
+
+        Ok(())
+    }
+
+    fn on_unload(&mut self) {
+        info!("Unloading plugin: {:?}", self.name());
+
+        match &mut self.client {
+            None => {}
+            Some(client) => match client {
+                PostgresClientEnum::Parallel(client) => client.join().unwrap(),
+                PostgresClientEnum::Simple(client) => {
+                    client.join().unwrap();
+                }
+            },
+        }
+    }
+
+    fn update_account(&mut self, account: ReplicaAccountInfoVersions, slot: u64) -> Result<()> {
+        match account {
+            ReplicaAccountInfoVersions::V0_0_1(account) => {
+                if let Some(accounts_selector) = &self.accounts_selector {
+                    if !accounts_selector.is_account_selected(account.pubkey, account.owner) {
+                        return Ok(());
+                    }
+                } else {
+                    return Ok(());
+                }
+
+                debug!(
+                    "Updating account {:?} with owner {:?} at slot {:?} using account selector {:?}",
+                    bs58::encode(account.pubkey).into_string(),
+                    bs58::encode(account.owner).into_string(),
+                    slot,
+                    self.accounts_selector.as_ref().unwrap()
+                );
+
+                match &mut self.client {
+                    None => {
+                        return Err(AccountsDbPluginError::Custom(Box::new(
+                            AccountsDbPluginPostgresError::DataStoreConnectionError {
+                                msg: "There is no connection to the PostgreSQL database."
+                                    .to_string(),
+                            },
+                        )));
+                    }
+                    Some(client) => {
+                        let result = match client {
+                            PostgresClientEnum::Parallel(client) => {
+                                client.update_account(account, slot)
+                            }
+                            PostgresClientEnum::Simple(client) => {
+                                client.update_account(account, slot)
+                            }
+                        };
+
+                        if let Err(err) = result {
+                            return Err(AccountsDbPluginError::AccountsUpdateError {
+                                msg: format!("Failed to persist the update of account to the PostgreSQL database. Error: {:?}", err)
+                            });
+                        }
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn update_slot_status(
+        &mut self,
+        slot: u64,
+        parent: Option<u64>,
+        status: SlotStatus,
+    ) -> Result<()> {
+        info!("Updating slot {:?} at with status {:?}", slot, status);
+
+        match &mut self.client {
+            None => {
+                return Err(AccountsDbPluginError::Custom(Box::new(
+                    AccountsDbPluginPostgresError::DataStoreConnectionError {
+                        msg: "There is no connection to the PostgreSQL database.".to_string(),
+                    },
+                )));
+            }
+            Some(client) => {
+                let result = match client {
+                    PostgresClientEnum::Parallel(client) => {
+                        client.update_slot_status(slot, parent, status)
+                    }
+                    PostgresClientEnum::Simple(client) => {
+                        client.update_slot_status(slot, parent, status)
+                    }
+                };
+
+                if let Err(err) = result {
+                    return Err(AccountsDbPluginError::SlotStatusUpdateError{
+                        msg: format!("Failed to persist the update of slot to the PostgreSQL database. Error: {:?}", err)
+                    });
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl AccountsDbPluginPostgres {
+    fn create_accounts_selector_from_config(config: &serde_json::Value) -> AccountsSelector {
+        let accounts_selector = &config["accounts_selector"];
+
+        if accounts_selector.is_null() {
+            AccountsSelector::default()
+        } else {
+            let accounts = &accounts_selector["accounts"];
+            let accounts: Vec<String> = if accounts.is_array() {
+                accounts
+                    .as_array()
+                    .unwrap()
+                    .iter()
+                    .map(|val| val.as_str().unwrap().to_string())
+                    .collect()
+            } else {
+                Vec::default()
+            };
+            let owners = &accounts_selector["owners"];
+            let owners: Vec<String> = if owners.is_array() {
+                owners
+                    .as_array()
+                    .unwrap()
+                    .iter()
+                    .map(|val| val.as_str().unwrap().to_string())
+                    .collect()
+            } else {
+                Vec::default()
+            };
+            AccountsSelector::new(&accounts, &owners)
+        }
+    }
+
+    pub fn new() -> Self {
+        AccountsDbPluginPostgres {
+            client: None,
+            accounts_selector: None,
+        }
+    }
+}
+
+#[no_mangle]
+#[allow(improper_ctypes_definitions)]
+/// # Safety
+///
+/// This function returns the AccountsDbPluginPostgres pointer as trait AccountsDbPlugin.
+pub unsafe extern "C" fn _create_plugin() -> *mut dyn AccountsDbPlugin {
+    let plugin = AccountsDbPluginPostgres::new();
+    let plugin: Box<dyn AccountsDbPlugin> = Box::new(plugin);
+    Box::into_raw(plugin)
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use {super::*, serde_json};
+
+    #[test]
+    fn test_accounts_selector_from_config() {
+        let config = "{\"accounts_selector\" : { \
+           \"owners\" : [\"9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin\"] \
+        }}";
+
+        let config: serde_json::Value = serde_json::from_str(config).unwrap();
+        AccountsDbPluginPostgres::create_accounts_selector_from_config(&config);
+    }
+}

accountsdb-plugin-postgres/src/lib.rs

@@ -0,0 +1,3 @@
+pub mod accounts_selector;
+pub mod accountsdb_plugin_postgres;
+pub mod postgres_client;

accountsdb-plugin-postgres/src/postgres_client.rs

@@ -0,0 +1,465 @@
+/// A concurrent implementation for writing accounts into the PostgreSQL in parallel.
+use {
+    crate::accountsdb_plugin_postgres::{
+        AccountsDbPluginPostgresConfig, AccountsDbPluginPostgresError,
+    },
+    chrono::Utc,
+    crossbeam_channel::{bounded, Receiver, RecvTimeoutError, Sender},
+    log::*,
+    postgres::{Client, NoTls, Statement},
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        AccountsDbPluginError, ReplicaAccountInfo, SlotStatus,
+    },
+    solana_metrics::datapoint_info,
+    solana_sdk::timing::AtomicInterval,
+    std::{
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc, Mutex,
+        },
+        thread::{self, Builder, JoinHandle},
+        time::Duration,
+    },
+};
+
+/// The maximum asynchronous requests allowed in the channel to avoid excessive
+/// memory usage. The downside -- calls after this threshold is reached can get blocked.
+const MAX_ASYNC_REQUESTS: usize = 10240;
+
+struct PostgresSqlClientWrapper {
+    client: Client,
+    update_account_stmt: Statement,
+}
+
+pub struct SimplePostgresClient {
+    client: Mutex<PostgresSqlClientWrapper>,
+}
+
+struct PostgresClientWorker {
+    client: SimplePostgresClient,
+}
+
+impl Eq for DbAccountInfo {}
+
+#[derive(Clone, PartialEq, Debug)]
+pub struct DbAccountInfo {
+    pub pubkey: Vec<u8>,
+    pub lamports: u64,
+    pub owner: Vec<u8>,
+    pub executable: bool,
+    pub rent_epoch: u64,
+    pub data: Vec<u8>,
+}
+
+impl DbAccountInfo {
+    fn new<T: ReadableAccountInfo>(account: &T) -> DbAccountInfo {
+        let data = account.data().to_vec();
+        Self {
+            pubkey: account.pubkey().to_vec(),
+            lamports: account.lamports(),
+            owner: account.owner().to_vec(),
+            executable: account.executable(),
+            rent_epoch: account.rent_epoch(),
+            data,
+        }
+    }
+}
+
+pub trait ReadableAccountInfo: Sized {
+    fn pubkey(&self) -> &[u8];
+    fn owner(&self) -> &[u8];
+    fn lamports(&self) -> u64;
+    fn executable(&self) -> bool;
+    fn rent_epoch(&self) -> u64;
+    fn data(&self) -> &[u8];
+}
+
+impl ReadableAccountInfo for DbAccountInfo {
+    fn pubkey(&self) -> &[u8] {
+        &self.pubkey
+    }
+
+    fn owner(&self) -> &[u8] {
+        &self.owner
+    }
+
+    fn lamports(&self) -> u64 {
+        self.lamports
+    }
+
+    fn executable(&self) -> bool {
+        self.executable
+    }
+
+    fn rent_epoch(&self) -> u64 {
+        self.rent_epoch
+    }
+
+    fn data(&self) -> &[u8] {
+        &self.data
+    }
+}
+
+impl<'a> ReadableAccountInfo for ReplicaAccountInfo<'a> {
+    fn pubkey(&self) -> &[u8] {
+        self.pubkey
+    }
+
+    fn owner(&self) -> &[u8] {
+        self.owner
+    }
+
+    fn lamports(&self) -> u64 {
+        self.lamports
+    }
+
+    fn executable(&self) -> bool {
+        self.executable
+    }
+
+    fn rent_epoch(&self) -> u64 {
+        self.rent_epoch
+    }
+
+    fn data(&self) -> &[u8] {
+        self.data
+    }
+}
+
+pub trait PostgresClient {
+    fn join(&mut self) -> thread::Result<()> {
+        Ok(())
+    }
+
+    fn update_account<T: ReadableAccountInfo>(
+        &mut self,
+        account: &T,
+        slot: u64,
+    ) -> Result<(), AccountsDbPluginError>;
+
+    fn update_slot_status(
+        &mut self,
+        slot: u64,
+        parent: Option<u64>,
+        status: SlotStatus,
+    ) -> Result<(), AccountsDbPluginError>;
+}
+
+impl SimplePostgresClient {
+    pub fn new(config: &AccountsDbPluginPostgresConfig) -> Result<Self, AccountsDbPluginError> {
+        let connection_str = format!("host={} user={}", config.host, config.user);
+        match Client::connect(&connection_str, NoTls) {
+            Err(err) => {
+                return Err(AccountsDbPluginError::Custom(Box::new(AccountsDbPluginPostgresError::DataStoreConnectionError {
+                    msg: format!(
+                        "Error in connecting to the PostgreSQL database: {:?} host: {:?} user: {:?} config: {:?}",
+                        err, config.host, config.user, connection_str
+                    ),
+                })));
+            }
+            Ok(mut client) => {
+                let result = client.prepare("INSERT INTO account (pubkey, slot, owner, lamports, executable, rent_epoch, data, updated_on) \
+                    VALUES ($1, $2, $3, $4, $5, $6, $7, $8) \
+                    ON CONFLICT (pubkey) DO UPDATE SET slot=$2, owner=$3, lamports=$4, executable=$5, rent_epoch=$6, \
+                    data=$7, updated_on=$8");
+
+                match result {
+                    Err(err) => {
+                        return Err(AccountsDbPluginError::Custom(Box::new(AccountsDbPluginPostgresError::DataSchemaError {
+                            msg: format!(
+                                "Error in preparing for the accounts update PostgreSQL database: {:?} host: {:?} user: {:?} config: {:?}",
+                                err, config.host, config.user, connection_str
+                            ),
+                        })));
+                    }
+                    Ok(update_account_stmt) => Ok(Self {
+                        client: Mutex::new(PostgresSqlClientWrapper {
+                            client,
+                            update_account_stmt,
+                        }),
+                    }),
+                }
+            }
+        }
+    }
+}
+
+impl PostgresClient for SimplePostgresClient {
+    fn update_account<T: ReadableAccountInfo>(
+        &mut self,
+        account: &T,
+        slot: u64,
+    ) -> Result<(), AccountsDbPluginError> {
+        trace!(
+            "Updating account {} with owner {} at slot {}",
+            bs58::encode(account.pubkey()).into_string(),
+            bs58::encode(account.owner()).into_string(),
+            slot,
+        );
+
+        let slot = slot as i64; // postgres only supports i64
+        let lamports = account.lamports() as i64;
+        let rent_epoch = account.rent_epoch() as i64;
+        let updated_on = Utc::now().naive_utc();
+        let client = self.client.get_mut().unwrap();
+        let result = client.client.query(
+            &client.update_account_stmt,
+            &[
+                &account.pubkey(),
+                &slot,
+                &account.owner(),
+                &lamports,
+                &account.executable(),
+                &rent_epoch,
+                &account.data(),
+                &updated_on,
+            ],
+        );
+
+        if let Err(err) = result {
+            let msg = format!(
+                "Failed to persist the update of account to the PostgreSQL database. Error: {:?}",
+                err
+            );
+            error!("{}", msg);
+            return Err(AccountsDbPluginError::AccountsUpdateError { msg });
+        }
+        Ok(())
+    }
+
+    fn update_slot_status(
+        &mut self,
+        slot: u64,
+        parent: Option<u64>,
+        status: SlotStatus,
+    ) -> Result<(), AccountsDbPluginError> {
+        info!("Updating slot {:?} at with status {:?}", slot, status);
+
+        let slot = slot as i64; // postgres only supports i64
+        let parent = parent.map(|parent| parent as i64);
+        let updated_on = Utc::now().naive_utc();
+        let status_str = status.as_str();
+        let client = self.client.get_mut().unwrap();
+
+        let result = match parent {
+                        Some(parent) => {
+                            client.client.execute(
+                                "INSERT INTO slot (slot, parent, status, updated_on) \
+                                VALUES ($1, $2, $3, $4) \
+                                ON CONFLICT (slot) DO UPDATE SET parent=$2, status=$3, updated_on=$4",
+                                &[
+                                    &slot,
+                                    &parent,
+                                    &status_str,
+                                    &updated_on,
+                                ],
+                            )
+                        }
+                        None => {
+                            client.client.execute(
+                                "INSERT INTO slot (slot, status, updated_on) \
+                                VALUES ($1, $2, $3) \
+                                ON CONFLICT (slot) DO UPDATE SET status=$2, updated_on=$3",
+                                &[
+                                    &slot,
+                                    &status_str,
+                                    &updated_on,
+                                ],
+                            )
+                        }
+                };
+
+        match result {
+            Err(err) => {
+                let msg = format!(
+                    "Failed to persist the update of slot to the PostgreSQL database. Error: {:?}",
+                    err
+                );
+                error!("{:?}", msg);
+                return Err(AccountsDbPluginError::SlotStatusUpdateError { msg });
+            }
+            Ok(rows) => {
+                assert_eq!(1, rows, "Expected one rows to be updated a time");
+            }
+        }
+
+        Ok(())
+    }
+}
+
+struct UpdateAccountRequest {
+    account: DbAccountInfo,
+    slot: u64,
+}
+
+struct UpdateSlotRequest {
+    slot: u64,
+    parent: Option<u64>,
+    slot_status: SlotStatus,
+}
+
+enum DbWorkItem {
+    UpdateAccount(UpdateAccountRequest),
+    UpdateSlot(UpdateSlotRequest),
+}
+
+impl PostgresClientWorker {
+    fn new(config: AccountsDbPluginPostgresConfig) -> Result<Self, AccountsDbPluginError> {
+        let client = SimplePostgresClient::new(&config)?;
+        Ok(PostgresClientWorker { client })
+    }
+
+    fn do_work(
+        &mut self,
+        receiver: Receiver<DbWorkItem>,
+        exit_worker: Arc<AtomicBool>,
+    ) -> Result<(), AccountsDbPluginError> {
+        while !exit_worker.load(Ordering::Relaxed) {
+            let work = receiver.recv_timeout(Duration::from_millis(500));
+
+            match work {
+                Ok(work) => match work {
+                    DbWorkItem::UpdateAccount(request) => {
+                        self.client.update_account(&request.account, request.slot)?;
+                    }
+                    DbWorkItem::UpdateSlot(request) => {
+                        self.client.update_slot_status(
+                            request.slot,
+                            request.parent,
+                            request.slot_status,
+                        )?;
+                    }
+                },
+                Err(err) => match err {
+                    RecvTimeoutError::Timeout => {
+                        continue;
+                    }
+                    _ => {
+                        error!("Error in receiving the item {:?}", err);
+                        break;
+                    }
+                },
+            }
+        }
+        Ok(())
+    }
+}
+pub struct ParallelPostgresClient {
+    workers: Vec<JoinHandle<Result<(), AccountsDbPluginError>>>,
+    exit_worker: Arc<AtomicBool>,
+    sender: Sender<DbWorkItem>,
+    last_report: AtomicInterval,
+}
+
+impl ParallelPostgresClient {
+    pub fn new(config: &AccountsDbPluginPostgresConfig) -> Result<Self, AccountsDbPluginError> {
+        let (sender, receiver) = bounded(MAX_ASYNC_REQUESTS);
+        let exit_worker = Arc::new(AtomicBool::new(false));
+        let mut workers = Vec::default();
+
+        for i in 0..config.threads.unwrap() {
+            let cloned_receiver = receiver.clone();
+            let exit_clone = exit_worker.clone();
+            let config = config.clone();
+            let worker = Builder::new()
+                .name(format!("worker-{}", i))
+                .spawn(move || -> Result<(), AccountsDbPluginError> {
+                    let mut worker = PostgresClientWorker::new(config)?;
+                    worker.do_work(cloned_receiver, exit_clone)?;
+                    Ok(())
+                })
+                .unwrap();
+
+            workers.push(worker);
+        }
+
+        Ok(Self {
+            last_report: AtomicInterval::default(),
+            workers,
+            exit_worker,
+            sender,
+        })
+    }
+}
+
+impl PostgresClient for ParallelPostgresClient {
+    fn join(&mut self) -> thread::Result<()> {
+        self.exit_worker.store(true, Ordering::Relaxed);
+        while !self.workers.is_empty() {
+            let worker = self.workers.pop();
+            if worker.is_none() {
+                break;
+            }
+            let worker = worker.unwrap();
+            let result = worker.join().unwrap();
+            if result.is_err() {
+                error!("The worker thread has failed: {:?}", result);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn update_account<T: ReadableAccountInfo>(
+        &mut self,
+        account: &T,
+        slot: u64,
+    ) -> Result<(), AccountsDbPluginError> {
+        if self.last_report.should_update(30000) {
+            datapoint_info!(
+                "postgres-plugin-stats",
+                ("message-queue-length", self.sender.len() as i64, i64),
+            );
+        }
+        if let Err(err) = self
+            .sender
+            .send(DbWorkItem::UpdateAccount(UpdateAccountRequest {
+                account: DbAccountInfo::new(account),
+                slot,
+            }))
+        {
+            return Err(AccountsDbPluginError::AccountsUpdateError {
+                msg: format!(
+                    "Failed to update the account {:?}, error: {:?}",
+                    bs58::encode(account.pubkey()).into_string(),
+                    err
+                ),
+            });
+        }
+        Ok(())
+    }
+
+    fn update_slot_status(
+        &mut self,
+        slot: u64,
+        parent: Option<u64>,
+        status: SlotStatus,
+    ) -> Result<(), AccountsDbPluginError> {
+        if let Err(err) = self.sender.send(DbWorkItem::UpdateSlot(UpdateSlotRequest {
+            slot,
+            parent,
+            slot_status: status,
+        })) {
+            return Err(AccountsDbPluginError::SlotStatusUpdateError {
+                msg: format!("Failed to update the slot {:?}, error: {:?}", slot, err),
+            });
+        }
+        Ok(())
+    }
+}
+
+pub struct PostgresClientBuilder {}
+
+impl PostgresClientBuilder {
+    pub fn build_pararallel_postgres_client(
+        config: &AccountsDbPluginPostgresConfig,
+    ) -> Result<ParallelPostgresClient, AccountsDbPluginError> {
+        ParallelPostgresClient::new(config)
+    }
+
+    pub fn build_simple_postgres_client(
+        config: &AccountsDbPluginPostgresConfig,
+    ) -> Result<SimplePostgresClient, AccountsDbPluginError> {
+        SimplePostgresClient::new(config)
+    }
+}

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.7.15"
+version = "1.8.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,18 +14,18 @@ crossbeam-channel = "0.4"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.5.0"
-solana-core = { path = "../core", version = "=1.7.15" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-gossip = { path = "../gossip", version = "=1.7.15" }
-solana-ledger = { path = "../ledger", version = "=1.7.15" }
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-measure = { path = "../measure", version = "=1.7.15" }
-solana-perf = { path = "../perf", version = "=1.7.15" }
-solana-poh = { path = "../poh", version = "=1.7.15" }
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
-solana-streamer = { path = "../streamer", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
+solana-core = { path = "../core", version = "=1.8.1" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-gossip = { path = "../gossip", version = "=1.8.1" }
+solana-ledger = { path = "../ledger", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-measure = { path = "../measure", version = "=1.8.1" }
+solana-perf = { path = "../perf", version = "=1.8.1" }
+solana-poh = { path = "../poh", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
+solana-streamer = { path = "../streamer", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -16,6 +16,7 @@ use solana_perf::packet::to_packets_chunked;
 use solana_poh::poh_recorder::{create_test_recorder, PohRecorder, WorkingBankEntry};
 use solana_runtime::{
     accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks,
+    cost_model::CostModel,
 };
 use solana_sdk::{
     hash::Hash,
@@ -27,7 +28,7 @@ use solana_sdk::{
 };
 use solana_streamer::socket::SocketAddrSpace;
 use std::{
-    sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex},
+    sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex, RwLock},
     thread::sleep,
     time::{Duration, Instant},
 };
@@ -231,6 +232,7 @@ fn main() {
             vote_receiver,
             None,
             replay_vote_sender,
+            Arc::new(RwLock::new(CostModel::default())),
         );
         poh_recorder.lock().unwrap().set_bank(&bank);
 

banks-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-client"
-version = "1.7.15"
+version = "1.8.1"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -15,16 +15,16 @@ borsh = "0.9.0"
 borsh-derive = "0.9.0"
 futures = "0.3"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "=1.7.15" }
-solana-program = { path = "../sdk/program", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
+solana-banks-interface = { path = "../banks-interface", version = "=1.8.1" }
+solana-program = { path = "../sdk/program", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
 tarpc = { version = "0.24.1", features = ["full"] }
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
-solana-banks-server = { path = "../banks-server", version = "=1.7.15" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
+solana-banks-server = { path = "../banks-server", version = "=1.8.1" }
 
 [lib]
 crate-type = ["lib"]

banks-client/src/lib.rs

@@ -385,7 +385,9 @@ mod tests {
         let message = Message::new(&[instruction], Some(&mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport = start_local_server(bank_forks, block_commitment_cache).await;
+            let client_transport =
+                start_local_server(bank_forks, block_commitment_cache, Duration::from_millis(1))
+                    .await;
             let mut banks_client = start_client(client_transport).await?;
 
             let recent_blockhash = banks_client.get_recent_blockhash().await?;
@@ -416,7 +418,9 @@ mod tests {
         let message = Message::new(&[instruction], Some(mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport = start_local_server(bank_forks, block_commitment_cache).await;
+            let client_transport =
+                start_local_server(bank_forks, block_commitment_cache, Duration::from_millis(1))
+                    .await;
             let mut banks_client = start_client(client_transport).await?;
             let (_, recent_blockhash, last_valid_block_height) = banks_client.get_fees().await?;
             let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);

banks-interface/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-interface"
-version = "1.7.15"
+version = "1.8.1"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,7 +12,7 @@ edition = "2018"
 [dependencies]
 mio = "0.7.6"
 serde = { version = "1.0.122", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
 tarpc = { version = "0.24.1", features = ["full"] }
 
 [dev-dependencies]

banks-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-server"
-version = "1.7.15"
+version = "1.8.1"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,10 +14,10 @@ bincode = "1.3.1"
 futures = "0.3"
 log = "0.4.11"
 mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "=1.7.15" }
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-metrics = { path = "../metrics", version = "=1.7.15" }
+solana-banks-interface = { path = "../banks-interface", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-metrics = { path = "../metrics", version = "=1.8.1" }
 tarpc = { version = "0.24.1", features = ["full"] }
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }

banks-server/src/banks_server.rs

@@ -12,6 +12,7 @@ use solana_sdk::{
     account::Account,
     clock::Slot,
     commitment_config::CommitmentLevel,
+    feature_set::FeatureSet,
     fee_calculator::FeeCalculator,
     hash::Hash,
     pubkey::Pubkey,
@@ -43,6 +44,7 @@ struct BanksServer {
     bank_forks: Arc<RwLock<BankForks>>,
     block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
     transaction_sender: Sender<TransactionInfo>,
+    poll_signature_status_sleep_duration: Duration,
 }
 
 impl BanksServer {
@@ -54,11 +56,13 @@ impl BanksServer {
         bank_forks: Arc<RwLock<BankForks>>,
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
         transaction_sender: Sender<TransactionInfo>,
+        poll_signature_status_sleep_duration: Duration,
     ) -> Self {
         Self {
             bank_forks,
             block_commitment_cache,
             transaction_sender,
+            poll_signature_status_sleep_duration,
         }
     }
 
@@ -81,6 +85,7 @@ impl BanksServer {
     fn new_loopback(
         bank_forks: Arc<RwLock<BankForks>>,
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+        poll_signature_status_sleep_duration: Duration,
     ) -> Self {
         let (transaction_sender, transaction_receiver) = channel();
         let bank = bank_forks.read().unwrap().working_bank();
@@ -95,7 +100,12 @@ impl BanksServer {
             .name("solana-bank-forks-client".to_string())
             .spawn(move || Self::run(server_bank_forks, transaction_receiver))
             .unwrap();
-        Self::new(bank_forks, block_commitment_cache, transaction_sender)
+        Self::new(
+            bank_forks,
+            block_commitment_cache,
+            transaction_sender,
+            poll_signature_status_sleep_duration,
+        )
     }
 
     fn slot(&self, commitment: CommitmentLevel) -> Slot {
@@ -120,7 +130,7 @@ impl BanksServer {
             .bank(commitment)
             .get_signature_status_with_blockhash(signature, blockhash);
         while status.is_none() {
-            sleep(Duration::from_millis(200)).await;
+            sleep(self.poll_signature_status_sleep_duration).await;
             let bank = self.bank(commitment);
             if bank.block_height() > last_valid_block_height {
                 break;
@@ -133,11 +143,11 @@ impl BanksServer {
 
 fn verify_transaction(
     transaction: &Transaction,
-    libsecp256k1_0_5_upgrade_enabled: bool,
+    feature_set: &Arc<FeatureSet>,
 ) -> transaction::Result<()> {
     if let Err(err) = transaction.verify() {
         Err(err)
-    } else if let Err(err) = transaction.verify_precompiles(libsecp256k1_0_5_upgrade_enabled) {
+    } else if let Err(err) = transaction.verify_precompiles(feature_set) {
         Err(err)
     } else {
         Ok(())
@@ -227,19 +237,13 @@ impl Banks for BanksServer {
         transaction: Transaction,
         commitment: CommitmentLevel,
     ) -> Option<transaction::Result<()>> {
-        if let Err(err) = verify_transaction(
-            &transaction,
-            self.bank(commitment).libsecp256k1_0_5_upgrade_enabled(),
-        ) {
+        if let Err(err) = verify_transaction(&transaction, &self.bank(commitment).feature_set) {
             return Some(Err(err));
         }
 
         let blockhash = &transaction.message.recent_blockhash;
         let last_valid_block_height = self
-            .bank_forks
-            .read()
-            .unwrap()
-            .root_bank()
+            .bank(commitment)
             .get_blockhash_last_valid_block_height(blockhash)
             .unwrap();
         let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
@@ -267,8 +271,13 @@ impl Banks for BanksServer {
 pub async fn start_local_server(
     bank_forks: Arc<RwLock<BankForks>>,
     block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+    poll_signature_status_sleep_duration: Duration,
 ) -> UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>> {
-    let banks_server = BanksServer::new_loopback(bank_forks, block_commitment_cache);
+    let banks_server = BanksServer::new_loopback(
+        bank_forks,
+        block_commitment_cache,
+        poll_signature_status_sleep_duration,
+    );
     let (client_transport, server_transport) = transport::channel::unbounded();
     let server = server::new(server::Config::default())
         .incoming(stream::once(future::ready(server_transport)))
@@ -303,8 +312,12 @@ pub async fn start_tcp_server(
 
             SendTransactionService::new(tpu_addr, &bank_forks, receiver);
 
-            let server =
-                BanksServer::new(bank_forks.clone(), block_commitment_cache.clone(), sender);
+            let server = BanksServer::new(
+                bank_forks.clone(),
+                block_commitment_cache.clone(),
+                sender,
+                Duration::from_millis(200),
+            );
             chan.respond_with(server.serve()).execute()
         })
         // Max 10 channels.

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.7.15"
+version = "1.8.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,23 +18,23 @@ rand = "0.7.0"
 rayon = "1.5.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-core = { path = "../core", version = "=1.7.15" }
-solana-genesis = { path = "../genesis", version = "=1.7.15" }
-solana-client = { path = "../client", version = "=1.7.15" }
-solana-exchange-program = { path = "../programs/exchange", version = "=1.7.15" }
-solana-faucet = { path = "../faucet", version = "=1.7.15" }
-solana-gossip = { path = "../gossip", version = "=1.7.15" }
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-metrics = { path = "../metrics", version = "=1.7.15" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.15" }
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-streamer = { path = "../streamer", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-core = { path = "../core", version = "=1.8.1" }
+solana-genesis = { path = "../genesis", version = "=1.8.1" }
+solana-client = { path = "../client", version = "=1.8.1" }
+solana-exchange-program = { path = "../programs/exchange", version = "=1.8.1" }
+solana-faucet = { path = "../faucet", version = "=1.8.1" }
+solana-gossip = { path = "../gossip", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-metrics = { path = "../metrics", version = "=1.8.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-streamer = { path = "../streamer", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "=1.7.15" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.8.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.7.15"
+version = "1.8.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-streamer = { path = "../streamer", version = "=1.7.15" }
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-streamer = { path = "../streamer", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.7.15"
+version = "1.8.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,24 +15,24 @@ log = "0.4.11"
 rayon = "1.5.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-core = { path = "../core", version = "=1.7.15" }
-solana-genesis = { path = "../genesis", version = "=1.7.15" }
-solana-client = { path = "../client", version = "=1.7.15" }
-solana-faucet = { path = "../faucet", version = "=1.7.15" }
-solana-gossip = { path = "../gossip", version = "=1.7.15" }
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-metrics = { path = "../metrics", version = "=1.7.15" }
-solana-measure = { path = "../measure", version = "=1.7.15" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.15" }
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-streamer = { path = "../streamer", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-core = { path = "../core", version = "=1.8.1" }
+solana-genesis = { path = "../genesis", version = "=1.8.1" }
+solana-client = { path = "../client", version = "=1.8.1" }
+solana-faucet = { path = "../faucet", version = "=1.8.1" }
+solana-gossip = { path = "../gossip", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-metrics = { path = "../metrics", version = "=1.8.1" }
+solana-measure = { path = "../measure", version = "=1.8.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-streamer = { path = "../streamer", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "=1.7.15" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.8.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

ci/do-audit.sh

@@ -45,5 +45,12 @@ cargo_audit_ignores=(
   # Blocked on jsonrpc removing dependency on unmaintained `websocket`
   # https://github.com/paritytech/jsonrpc/issues/605
   --ignore RUSTSEC-2021-0079
+
+  # chrono: Potential segfault in `localtime_r` invocations
+  #
+  # Blocked due to no safe upgrade
+  # https://github.com/chronotope/chrono/issues/499
+  --ignore RUSTSEC-2020-0159
+
 )
 scripts/cargo-for-all-lock-files.sh stable audit "${cargo_audit_ignores[@]}"

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.7.15"
+version = "1.8.1"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,8 +12,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
 thiserror = "1.0.21"
 tiny-bip39 = "0.8.1"
 uriparse = "0.6.3"

clap-utils/src/keypair.rs

@@ -1,3 +1,14 @@
+//! Loading signers and keypairs from the command line.
+//!
+//! This module contains utilities for loading [Signer]s and [Keypair]s from
+//! standard signing sources, from the command line, as in the Solana CLI.
+//!
+//! The key function here is [`signer_from_path`], which loads a `Signer` from
+//! one of several possible sources by interpreting a "path" command line
+//! argument. Its documentation includes a description of all possible signing
+//! sources supported by the Solana CLI. Many other functions here are
+//! variations on, or delegate to, `signer_from_path`.
+
 use {
     crate::{
         input_parsers::{pubkeys_sigs_of, STDOUT_OUTFILE_TOKEN},
@@ -92,14 +103,56 @@ impl CliSignerInfo {
     }
 }
 
+/// A command line argument that loads a default signer in absence of other signers.
+///
+/// This type manages a default signing source which may be overridden by other
+/// signing sources via its [`generate_unique_signers`] method.
+///
+/// [`generate_unique_signers`]: DefaultSigner::generate_unique_signers
+///
+/// `path` is a signing source as documented by [`signer_from_path`], and
+/// `arg_name` is the name of its [clap] command line argument, which is passed
+/// to `signer_from_path` as its `keypair_name` argument.
 #[derive(Debug, Default)]
 pub struct DefaultSigner {
+    /// The name of the signers command line argument.
     pub arg_name: String,
+    /// The signing source.
     pub path: String,
     is_path_checked: RefCell<bool>,
 }
 
 impl DefaultSigner {
+    /// Create a new `DefaultSigner`.
+    ///
+    /// `path` is a signing source as documented by [`signer_from_path`], and
+    /// `arg_name` is the name of its [clap] command line argument, which is
+    /// passed to `signer_from_path` as its `keypair_name` argument.
+    ///
+    /// [clap]: https://docs.rs/clap
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use clap::{App, Arg, value_t_or_exit};
+    /// use solana_clap_utils::keypair::DefaultSigner;
+    /// use solana_clap_utils::offline::OfflineArgs;
+    ///
+    /// let clap_app = App::new("my-program")
+    ///     // The argument we'll parse as a signer "path"
+    ///     .arg(Arg::with_name("keypair")
+    ///         .required(true)
+    ///         .help("The default signer"))
+    ///     .offline_args();
+    ///
+    /// let clap_matches = clap_app.get_matches();
+    /// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+    ///
+    /// let default_signer = DefaultSigner::new("keypair", &keypair_str);
+    /// # assert!(default_signer.arg_name.len() > 0);
+    /// assert_eq!(default_signer.path, keypair_str);
+    /// # Ok::<(), Box<dyn std::error::Error>>(())
+    /// ```
     pub fn new<AN: AsRef<str>, P: AsRef<str>>(arg_name: AN, path: P) -> Self {
         let arg_name = arg_name.as_ref().to_string();
         let path = path.as_ref().to_string();
@@ -134,6 +187,57 @@ impl DefaultSigner {
         Ok(&self.path)
     }
 
+    /// Generate a unique set of signers, possibly excluding this default signer.
+    ///
+    /// This function allows a command line application to have a default
+    /// signer, perhaps representing a default wallet, but to override that
+    /// signer and instead sign with one or more other signers.
+    ///
+    /// `bulk_signers` is a vector of signers, all of which are optional. If any
+    /// of those signers is `None`, then the default signer will be loaded; if
+    /// all of those signers are `Some`, then the default signer will not be
+    /// loaded.
+    ///
+    /// The returned value includes all of the `bulk_signers` that were not
+    /// `None`, and maybe the default signer, if it was loaded.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use clap::{App, Arg, value_t_or_exit};
+    /// use solana_clap_utils::keypair::{DefaultSigner, signer_from_path};
+    /// use solana_clap_utils::offline::OfflineArgs;
+    /// use solana_sdk::signer::Signer;
+    ///
+    /// let clap_app = App::new("my-program")
+    ///     // The argument we'll parse as a signer "path"
+    ///     .arg(Arg::with_name("keypair")
+    ///         .required(true)
+    ///         .help("The default signer"))
+    ///     .arg(Arg::with_name("payer")
+    ///         .long("payer")
+    ///         .help("The account paying for the transaction"))
+    ///     .offline_args();
+    ///
+    /// let mut wallet_manager = None;
+    ///
+    /// let clap_matches = clap_app.get_matches();
+    /// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+    /// let maybe_payer = clap_matches.value_of("payer");
+    ///
+    /// let default_signer = DefaultSigner::new("keypair", &keypair_str);
+    /// let maybe_payer_signer = maybe_payer.map(|payer| {
+    ///     signer_from_path(&clap_matches, payer, "payer", &mut wallet_manager)
+    /// }).transpose()?;
+    /// let bulk_signers = vec![maybe_payer_signer];
+    ///
+    /// let unique_signers = default_signer.generate_unique_signers(
+    ///     bulk_signers,
+    ///     &clap_matches,
+    ///     &mut wallet_manager,
+    /// )?;
+    /// # Ok::<(), Box<dyn std::error::Error>>(())
+    /// ```
     pub fn generate_unique_signers(
         &self,
         bulk_signers: Vec<Option<Box<dyn Signer>>>,
@@ -158,6 +262,45 @@ impl DefaultSigner {
         })
     }
 
+    /// Loads the default [Signer] from one of several possible sources.
+    ///
+    /// The `path` is not strictly a file system path, but is interpreted as
+    /// various types of _signing source_, depending on its format, one of which
+    /// is a path to a keypair file. Some sources may require user interaction
+    /// in the course of calling this function.
+    ///
+    /// This simply delegates to the [`signer_from_path`] free function, passing
+    /// it the `DefaultSigner`s `path` and `arg_name` fields as the `path` and
+    /// `keypair_name` arguments.
+    ///
+    /// See the [`signer_from_path`] free function for full documentation of how
+    /// this function interprets its arguments.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use clap::{App, Arg, value_t_or_exit};
+    /// use solana_clap_utils::keypair::DefaultSigner;
+    /// use solana_clap_utils::offline::OfflineArgs;
+    ///
+    /// let clap_app = App::new("my-program")
+    ///     // The argument we'll parse as a signer "path"
+    ///     .arg(Arg::with_name("keypair")
+    ///         .required(true)
+    ///         .help("The default signer"))
+    ///     .offline_args();
+    ///
+    /// let clap_matches = clap_app.get_matches();
+    /// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+    /// let default_signer = DefaultSigner::new("keypair", &keypair_str);
+    /// let mut wallet_manager = None;
+    ///
+    /// let signer = default_signer.signer_from_path(
+    ///     &clap_matches,
+    ///     &mut wallet_manager,
+    /// )?;
+    /// # Ok::<(), Box<dyn std::error::Error>>(())
+    /// ```
     pub fn signer_from_path(
         &self,
         matches: &ArgMatches,
@@ -166,6 +309,51 @@ impl DefaultSigner {
         signer_from_path(matches, self.path()?, &self.arg_name, wallet_manager)
     }
 
+    /// Loads the default [Signer] from one of several possible sources.
+    ///
+    /// The `path` is not strictly a file system path, but is interpreted as
+    /// various types of _signing source_, depending on its format, one of which
+    /// is a path to a keypair file. Some sources may require user interaction
+    /// in the course of calling this function.
+    ///
+    /// This simply delegates to the [`signer_from_path_with_config`] free
+    /// function, passing it the `DefaultSigner`s `path` and `arg_name` fields
+    /// as the `path` and `keypair_name` arguments.
+    ///
+    /// See the [`signer_from_path`] free function for full documentation of how
+    /// this function interprets its arguments.
+    ///
+    /// # Examples
+    ///
+    /// ```no_run
+    /// use clap::{App, Arg, value_t_or_exit};
+    /// use solana_clap_utils::keypair::{SignerFromPathConfig, DefaultSigner};
+    /// use solana_clap_utils::offline::OfflineArgs;
+    ///
+    /// let clap_app = App::new("my-program")
+    ///     // The argument we'll parse as a signer "path"
+    ///     .arg(Arg::with_name("keypair")
+    ///         .required(true)
+    ///         .help("The default signer"))
+    ///     .offline_args();
+    ///
+    /// let clap_matches = clap_app.get_matches();
+    /// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+    /// let default_signer = DefaultSigner::new("keypair", &keypair_str);
+    /// let mut wallet_manager = None;
+    ///
+    /// // Allow pubkey signers without accompanying signatures
+    /// let config = SignerFromPathConfig {
+    ///     allow_null_signer: true,
+    /// };
+    ///
+    /// let signer = default_signer.signer_from_path_with_config(
+    ///     &clap_matches,
+    ///     &mut wallet_manager,
+    ///     &config,
+    /// )?;
+    /// # Ok::<(), Box<dyn std::error::Error>>(())
+    /// ```
     pub fn signer_from_path_with_config(
         &self,
         matches: &ArgMatches,
@@ -258,6 +446,15 @@ pub(crate) fn parse_signer_source<S: AsRef<str>>(
     let source = {
         #[cfg(target_family = "windows")]
         {
+            // trim matched single-quotes since cmd.exe won't
+            let mut source = source;
+            while let Some(trimmed) = source.strip_prefix('\'') {
+                source = if let Some(trimmed) = trimmed.strip_suffix('\'') {
+                    trimmed
+                } else {
+                    break;
+                }
+            }
             source.replace("\\", "/")
         }
         #[cfg(not(target_family = "windows"))]
@@ -324,19 +521,167 @@ pub fn presigner_from_pubkey_sigs(
     })
 }
 
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct SignerFromPathConfig {
     pub allow_null_signer: bool,
 }
 
-impl Default for SignerFromPathConfig {
-    fn default() -> Self {
-        Self {
-            allow_null_signer: false,
-        }
-    }
-}
-
+/// Loads a [Signer] from one of several possible sources.
+///
+/// The `path` is not strictly a file system path, but is interpreted as various
+/// types of _signing source_, depending on its format, one of which is a path
+/// to a keypair file. Some sources may require user interaction in the course
+/// of calling this function.
+///
+/// The result of this function is a boxed object of the [Signer] trait. To load
+/// a concrete [Keypair], use the [keypair_from_path] function, though note that
+/// it does not support all signer sources.
+///
+/// The `matches` argument is the same set of parsed [clap] matches from which
+/// `path` was parsed. It is used to parse various additional command line
+/// arguments, depending on which signing source is requested, as described
+/// below in "Signing sources".
+///
+/// [clap]: https//docs.rs/clap
+///
+/// The `keypair_name` argument is the "name" of the signer, and is typically
+/// the name of the clap argument from which the `path` argument was parsed,
+/// like "keypair", "from", or "fee-payer". It is used solely for interactively
+/// prompting the user, either when entering seed phrases or selecting from
+/// multiple hardware wallets.
+///
+/// The `wallet_manager` is used for establishing connections to a hardware
+/// device such as Ledger. If `wallet_manager` is a reference to `None`, and a
+/// hardware signer is requested, then this function will attempt to create a
+/// wallet manager, assigning it to the mutable `wallet_manager` reference. This
+/// argument is typically a reference to `None`.
+///
+/// # Signing sources
+///
+/// The `path` argument can simply be a path to a keypair file, but it may also
+/// be interpreted in several other ways, in the following order.
+///
+/// Firstly, the `path` argument may be interpreted as a [URI], with the URI
+/// scheme indicating where to load the signer from. If it parses as a URI, then
+/// the following schemes are supported:
+///
+/// - `file:` &mdash; Read the keypair from a JSON keypair file. The path portion
+///    of the URI is the file path.
+///
+/// - `stdin:` &mdash; Read the keypair from stdin, in the JSON format used by
+///   the keypair file.
+///
+///   Non-scheme parts of the URI are ignored.
+///
+/// - `prompt:` &mdash; The user will be prompted at the command line
+///   for their seed phrase and passphrase.
+///
+///   In this URI the [query string][qs] may contain zero or one of the
+///   following key/value pairs that determine the [BIP44 derivation path][dp]
+///   of the private key from the seed:
+///
+///   - `key` &mdash; In this case the value is either one or two numerical
+///     indexes separated by a slash, which represent the "account", and
+///     "change" components of the BIP44 derivation path. Example: `key=0/0`.
+///
+///   - `full-path` &mdash; In this case the value is a full derivation path,
+///     and the user is responsible for ensuring it is correct. Example:
+///     `full-path=m/44/501/0/0/0`.
+///
+///   If neither is provided, then the default derivation path is used.
+///
+///   Note that when specifying derivation paths, this routine will convert all
+///   indexes into ["hardened"] indexes, even if written as "normal" indexes.
+///
+///   Other components of the URI besides the scheme and query string are ignored.
+///
+///   If the "skip_seed_phrase_validation" argument, as defined in
+///   [SKIP_SEED_PHRASE_VALIDATION_ARG] is found in `matches`, then the keypair
+///   seed will be generated directly from the seed phrase, without parsing or
+///   validating it as a BIP39 seed phrase. This allows the use of non-BIP39 seed
+///   phrases.
+///
+/// - `usb:` &mdash; Use a USB hardware device as the signer. In this case, the
+///   URI host indicates the device type, and is required. The only currently valid host
+///   value is "ledger".
+///
+///   Optionally, the first segment of the URI path indicates the base-58
+///   encoded pubkey of the wallet, and the "account" and "change" indices of
+///   the derivation path can be specified with the `key=` query parameter, as
+///   with the `prompt:` URI.
+///
+///   Examples:
+///
+///   - `usb://ledger`
+///   - `usb://ledger?key=0/0`
+///   - `usb://ledger/9rPVSygg3brqghvdZ6wsL2i5YNQTGhXGdJzF65YxaCQd`
+///   - `usb://ledger/9rPVSygg3brqghvdZ6wsL2i5YNQTGhXGdJzF65YxaCQd?key=0/0`
+///
+/// Next the `path` argument may be one of the following strings:
+///
+/// - `-` &mdash; Read the keypair from stdin. This is the same as the `stdin:`
+///   URI scheme.
+///
+/// - `ASK` &mdash; The user will be prompted at the command line for their seed
+///   phrase and passphrase. _This uses a legacy key derivation method and should
+///   usually be avoided in favor of `prompt:`._
+///
+/// Next, if the `path` argument parses as a base-58 public key, then the signer
+/// is created without a private key, but with presigned signatures, each parsed
+/// from the additional command line arguments, provided by the `matches`
+/// argument.
+///
+/// In this case, the remaining command line arguments are searched for clap
+/// arguments named "signer", as defined by [SIGNER_ARG], and each is parsed as
+/// a key-value pair of the form "pubkey=signature", where `pubkey` is the same
+/// base-58 public key, and `signature` is a serialized signature produced by
+/// the corresponding keypair. One of the "signer" signatures must be for the
+/// pubkey specified in `path` or this function will return an error; unless the
+/// "sign_only" clap argument, as defined by [SIGN_ONLY_ARG], is present in
+/// `matches`, in which case the signer will be created with no associated
+/// signatures.
+///
+/// Finally, if `path`, interpreted as a file path, represents a file on disk,
+/// then the signer is created by reading that file as a JSON-serialized
+/// keypair. This is the same as the `file:` URI scheme.
+///
+/// [qs]: https://en.wikipedia.org/wiki/Query_string
+/// [dp]: https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki
+/// [URI]: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
+/// ["hardened"]: https://wiki.trezor.io/Hardened_and_non-hardened_derivation
+///
+/// # Examples
+///
+/// This shows a reasonable way to set up clap to parse all possible signer
+/// sources. Note the use of the [`OfflineArgs::offline_args`] method to add
+/// correct clap definitions of the `--signer` and `--sign-only` arguments, as
+/// required by the base-58 pubkey offline signing method.
+///
+/// [`OfflineArgs::offline_args`]: crate::offline::OfflineArgs::offline_args
+///
+/// ```no_run
+/// use clap::{App, Arg, value_t_or_exit};
+/// use solana_clap_utils::keypair::signer_from_path;
+/// use solana_clap_utils::offline::OfflineArgs;
+///
+/// let clap_app = App::new("my-program")
+///     // The argument we'll parse as a signer "path"
+///     .arg(Arg::with_name("keypair")
+///         .required(true)
+///         .help("The default signer"))
+///     .offline_args();
+///
+/// let clap_matches = clap_app.get_matches();
+/// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+/// let mut wallet_manager = None;
+/// let signer = signer_from_path(
+///     &clap_matches,
+///     &keypair_str,
+///     "keypair",
+///     &mut wallet_manager,
+/// )?;
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
 pub fn signer_from_path(
     matches: &ArgMatches,
     path: &str,
@@ -347,6 +692,63 @@ pub fn signer_from_path(
     signer_from_path_with_config(matches, path, keypair_name, wallet_manager, &config)
 }
 
+/// Loads a [Signer] from one of several possible sources.
+///
+/// The `path` is not strictly a file system path, but is interpreted as various
+/// types of _signing source_, depending on its format, one of which is a path
+/// to a keypair file. Some sources may require user interaction in the course
+/// of calling this function.
+///
+/// This is the same as [`signer_from_path`] except that it additionaolly
+/// accepts a [`SignerFromPathConfig`] argument.
+///
+/// If the `allow_null_signer` field of `config` is `true`, then pubkey signers
+/// are allowed to have zero associated signatures via additional "signer"
+/// command line arguments. It the same effect as if the "sign_only" clap
+/// argument is present.
+///
+/// See [`signer_from_path`] for full documentation of how this function
+/// interprets its arguments.
+///
+/// # Examples
+///
+/// This shows a reasonable way to set up clap to parse all possible signer
+/// sources. Note the use of the [`OfflineArgs::offline_args`] method to add
+/// correct clap definitions of the `--signer` and `--sign-only` arguments, as
+/// required by the base-58 pubkey offline signing method.
+///
+/// [`OfflineArgs::offline_args`]: crate::offline::OfflineArgs::offline_args
+///
+/// ```no_run
+/// use clap::{App, Arg, value_t_or_exit};
+/// use solana_clap_utils::keypair::{signer_from_path_with_config, SignerFromPathConfig};
+/// use solana_clap_utils::offline::OfflineArgs;
+///
+/// let clap_app = App::new("my-program")
+///     // The argument we'll parse as a signer "path"
+///     .arg(Arg::with_name("keypair")
+///         .required(true)
+///         .help("The default signer"))
+///     .offline_args();
+///
+/// let clap_matches = clap_app.get_matches();
+/// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+/// let mut wallet_manager = None;
+///
+/// // Allow pubkey signers without accompanying signatures
+/// let config = SignerFromPathConfig {
+///     allow_null_signer: true,
+/// };
+///
+/// let signer = signer_from_path_with_config(
+///     &clap_matches,
+///     &keypair_str,
+///     "keypair",
+///     &mut wallet_manager,
+///     &config,
+/// )?;
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
 pub fn signer_from_path_with_config(
     matches: &ArgMatches,
     path: &str,
@@ -417,6 +819,43 @@ pub fn signer_from_path_with_config(
     }
 }
 
+/// Loads the pubkey of a [Signer] from one of several possible sources.
+///
+/// The `path` is not strictly a file system path, but is interpreted as various
+/// types of _signing source_, depending on its format, one of which is a path
+/// to a keypair file. Some sources may require user interaction in the course
+/// of calling this function.
+///
+/// The only difference between this function and [`signer_from_path`] is in the
+/// case of a "pubkey" path: this function does not require that accompanying
+/// command line arguments contain an offline signature.
+///
+/// See [`signer_from_path`] for full documentation of how this function
+/// interprets its arguments.
+///
+/// # Examples
+///
+/// ```no_run
+/// use clap::{App, Arg, value_t_or_exit};
+/// use solana_clap_utils::keypair::pubkey_from_path;
+///
+/// let clap_app = App::new("my-program")
+///     // The argument we'll parse as a signer "path"
+///     .arg(Arg::with_name("keypair")
+///         .required(true)
+///         .help("The default signer"));
+///
+/// let clap_matches = clap_app.get_matches();
+/// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+/// let mut wallet_manager = None;
+/// let pubkey = pubkey_from_path(
+///     &clap_matches,
+///     &keypair_str,
+///     "keypair",
+///     &mut wallet_manager,
+/// )?;
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
 pub fn pubkey_from_path(
     matches: &ArgMatches,
     path: &str,
@@ -516,7 +955,46 @@ pub fn prompt_passphrase(prompt: &str) -> Result<String, Box<dyn error::Error>>
     Ok(passphrase)
 }
 
-/// Parses a path into a SignerSource and returns a Keypair for supporting SignerSourceKinds
+/// Loads a [Keypair] from one of several possible sources.
+///
+/// The `path` is not strictly a file system path, but is interpreted as various
+/// types of _signing source_, depending on its format, one of which is a path
+/// to a keypair file. Some sources may require user interaction in the course
+/// of calling this function.
+///
+/// This is the same as [`signer_from_path`] except that it only supports
+/// signing sources that can result in a [Keypair]: prompt for seed phrase,
+/// keypair file, and stdin.
+///
+/// If `confirm_pubkey` is `true` then after deriving the pubkey, the user will
+/// be prompted to confirm that the pubkey is as expected.
+///
+/// See [`signer_from_path`] for full documentation of how this function
+/// interprets its arguments.
+///
+/// # Examples
+///
+/// ```no_run
+/// use clap::{App, Arg, value_t_or_exit};
+/// use solana_clap_utils::keypair::keypair_from_path;
+///
+/// let clap_app = App::new("my-program")
+///     // The argument we'll parse as a signer "path"
+///     .arg(Arg::with_name("keypair")
+///         .required(true)
+///         .help("The default signer"));
+///
+/// let clap_matches = clap_app.get_matches();
+/// let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+///
+/// let signer = keypair_from_path(
+///     &clap_matches,
+///     &keypair_str,
+///     "keypair",
+///     false,
+/// )?;
+/// # Ok::<(), Box<dyn std::error::Error>>(())
+/// ```
 pub fn keypair_from_path(
     matches: &ArgMatches,
     path: &str,
@@ -566,9 +1044,10 @@ pub fn keypair_from_path(
     }
 }
 
-/// Reads user input from stdin to retrieve a seed phrase and passphrase for keypair derivation
-/// Optionally skips validation of seed phrase
-/// Optionally confirms recovered public key
+/// Reads user input from stdin to retrieve a seed phrase and passphrase for keypair derivation.
+///
+/// Optionally skips validation of seed phrase. Optionally confirms recovered
+/// public key.
 pub fn keypair_from_seed_phrase(
     keypair_name: &str,
     skip_validation: bool,
@@ -645,9 +1124,13 @@ fn sanitize_seed_phrase(seed_phrase: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::offline::OfflineArgs;
+    use clap::{value_t_or_exit, App, Arg};
     use solana_remote_wallet::locator::Manufacturer;
+    use solana_remote_wallet::remote_wallet::initialize_wallet_manager;
+    use solana_sdk::signer::keypair::write_keypair_file;
     use solana_sdk::system_instruction;
-    use tempfile::NamedTempFile;
+    use tempfile::{NamedTempFile, TempDir};
 
     #[test]
     fn test_sanitize_seed_phrase() {
@@ -806,4 +1289,41 @@ mod tests {
             } if p == relative_path_str)
         );
     }
+
+    #[test]
+    fn signer_from_path_with_file() -> Result<(), Box<dyn std::error::Error>> {
+        let dir = TempDir::new()?;
+        let dir = dir.path();
+        let keypair_path = dir.join("id.json");
+        let keypair_path_str = keypair_path.to_str().expect("utf-8");
+
+        let keypair = Keypair::new();
+        write_keypair_file(&keypair, &keypair_path)?;
+
+        let args = vec!["program", keypair_path_str];
+
+        let clap_app = App::new("my-program")
+            .arg(
+                Arg::with_name("keypair")
+                    .required(true)
+                    .help("The signing keypair"),
+            )
+            .offline_args();
+
+        let clap_matches = clap_app.get_matches_from(args);
+        let keypair_str = value_t_or_exit!(clap_matches, "keypair", String);
+
+        let wallet_manager = initialize_wallet_manager()?;
+
+        let signer = signer_from_path(
+            &clap_matches,
+            &keypair_str,
+            "signer",
+            &mut Some(wallet_manager),
+        )?;
+
+        assert_eq!(keypair.pubkey(), signer.pubkey());
+
+        Ok(())
+    }
 }

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.15"
+version = "1.8.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

cli-output/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.15"
+version = "1.8.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -20,12 +20,12 @@ indicatif = "0.15.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.15" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-client = { path = "../client", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.15" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.15" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.8.1" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-client = { path = "../client", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.8.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.8.1" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 
 [package.metadata.docs.rs]

cli-output/src/cli_output.rs

@@ -576,7 +576,7 @@ impl fmt::Display for CliValidators {
         for (version, info) in self.stake_by_version.iter() {
             writeln!(
                 f,
-                "{:<8} - {:3} current validators ({:>5.2}%){}",
+                "{:<8} - {:4} current validators ({:>5.2}%){}",
                 version,
                 info.current_validators,
                 100. * info.current_active_stake as f64 / self.total_active_stake as f64,

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.15"
+version = "1.8.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -29,30 +29,30 @@ reqwest = { version = "0.11.2", default-features = false, features = ["blocking"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.15" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.7.15" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-cli-config = { path = "../cli-config", version = "=1.7.15" }
-solana-cli-output = { path = "../cli-output", version = "=1.7.15" }
-solana-client = { path = "../client", version = "=1.7.15" }
-solana-config-program = { path = "../programs/config", version = "=1.7.15" }
-solana-faucet = { path = "../faucet", version = "=1.7.15" }
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.15" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.8.1" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.8.1" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-cli-config = { path = "../cli-config", version = "=1.8.1" }
+solana-cli-output = { path = "../cli-output", version = "=1.8.1" }
+solana-client = { path = "../client", version = "=1.8.1" }
+solana-config-program = { path = "../programs/config", version = "=1.8.1" }
+solana-faucet = { path = "../faucet", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.1" }
 solana_rbpf = "=0.2.11"
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.15" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.8.1" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 thiserror = "1.0.21"
 tiny-bip39 = "0.8.1"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "=1.7.15" }
-solana-streamer = { path = "../streamer", version = "=1.7.15" }
+solana-core = { path = "../core", version = "=1.8.1" }
+solana-streamer = { path = "../streamer", version = "=1.8.1" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cluster_query.rs

@@ -1721,7 +1721,7 @@ pub fn process_show_stakes(
                 // Filter by `StakeState::Stake(_, _)`
                 rpc_filter::RpcFilterType::Memcmp(rpc_filter::Memcmp {
                     offset: 0,
-                    bytes: rpc_filter::MemcmpEncodedBytes::Binary(
+                    bytes: rpc_filter::MemcmpEncodedBytes::Base58(
                         bs58::encode([2, 0, 0, 0]).into_string(),
                     ),
                     encoding: Some(rpc_filter::MemcmpEncoding::Binary),
@@ -1729,7 +1729,7 @@ pub fn process_show_stakes(
                 // Filter by `Delegation::voter_pubkey`, which begins at byte offset 124
                 rpc_filter::RpcFilterType::Memcmp(rpc_filter::Memcmp {
                     offset: 124,
-                    bytes: rpc_filter::MemcmpEncodedBytes::Binary(
+                    bytes: rpc_filter::MemcmpEncodedBytes::Base58(
                         vote_account_pubkeys[0].to_string(),
                     ),
                     encoding: Some(rpc_filter::MemcmpEncoding::Binary),

cli/src/feature.rs

@@ -222,43 +222,63 @@ pub fn process_feature_subcommand(
     }
 }
 
-fn active_stake_by_feature_set(rpc_client: &RpcClient) -> Result<HashMap<u32, f64>, ClientError> {
+fn feature_set_stats(rpc_client: &RpcClient) -> Result<HashMap<u32, (f64, f32)>, ClientError> {
     // Validator identity -> feature set
-    let feature_set_map = rpc_client
+    let feature_sets = rpc_client
         .get_cluster_nodes()?
         .into_iter()
-        .map(|contact_info| (contact_info.pubkey, contact_info.feature_set))
-        .collect::<HashMap<_, _>>();
+        .map(|contact_info| {
+            (
+                contact_info.pubkey,
+                contact_info.feature_set,
+                contact_info.rpc.is_some(),
+            )
+        })
+        .collect::<Vec<_>>();
 
     let vote_accounts = rpc_client.get_vote_accounts()?;
 
-    let total_active_stake: u64 = vote_accounts
-        .current
+    let mut total_active_stake: u64 = vote_accounts
+        .delinquent
         .iter()
-        .chain(vote_accounts.delinquent.iter())
         .map(|vote_account| vote_account.activated_stake)
         .sum();
 
-    // Sum all active stake by feature set
-    let mut active_stake_by_feature_set: HashMap<u32, u64> = HashMap::new();
-    for vote_account in vote_accounts.current {
-        if let Some(Some(feature_set)) = feature_set_map.get(&vote_account.node_pubkey) {
-            *active_stake_by_feature_set.entry(*feature_set).or_default() +=
-                vote_account.activated_stake;
-        } else {
-            *active_stake_by_feature_set
-                .entry(0 /* "unknown" */)
-                .or_default() += vote_account.activated_stake;
+    let vote_stakes = vote_accounts
+        .current
+        .into_iter()
+        .map(|vote_account| {
+            total_active_stake += vote_account.activated_stake;
+            (vote_account.node_pubkey, vote_account.activated_stake)
+        })
+        .collect::<HashMap<_, _>>();
+
+    let mut feature_set_stats: HashMap<u32, (u64, u32)> = HashMap::new();
+    let mut total_rpc_nodes = 0;
+    for (node_id, feature_set, is_rpc) in feature_sets {
+        let feature_set = feature_set.unwrap_or(0);
+        let feature_set_entry = feature_set_stats.entry(feature_set).or_default();
+
+        if let Some(vote_stake) = vote_stakes.get(&node_id) {
+            feature_set_entry.0 += *vote_stake;
+        }
+
+        if is_rpc {
+            feature_set_entry.1 += 1;
+            total_rpc_nodes += 1;
         }
     }
 
-    Ok(active_stake_by_feature_set
+    Ok(feature_set_stats
         .into_iter()
-        .map(|(feature_set, active_stake)| {
-            (
-                feature_set,
-                active_stake as f64 * 100. / total_active_stake as f64,
-            )
+        .filter_map(|(feature_set, (active_stake, is_rpc))| {
+            let active_stake = active_stake as f64 * 100. / total_active_stake as f64;
+            let is_rpc = is_rpc as f32 * 100. / total_rpc_nodes as f32;
+            if active_stake >= 0.001 || is_rpc >= 0.001 {
+                Some((feature_set, (active_stake, is_rpc)))
+            } else {
+                None
+            }
         })
         .collect())
 }
@@ -267,50 +287,93 @@ fn active_stake_by_feature_set(rpc_client: &RpcClient) -> Result<HashMap<u32, f6
 fn feature_activation_allowed(rpc_client: &RpcClient, quiet: bool) -> Result<bool, ClientError> {
     let my_feature_set = solana_version::Version::default().feature_set;
 
-    let active_stake_by_feature_set = active_stake_by_feature_set(rpc_client)?;
+    let feature_set_stats = feature_set_stats(rpc_client)?;
 
-    let feature_activation_allowed = active_stake_by_feature_set
+    let (stake_allowed, rpc_allowed) = feature_set_stats
         .get(&my_feature_set)
-        .map(|percentage| *percentage >= 95.)
-        .unwrap_or(false);
+        .map(|(stake_percent, rpc_percent)| (*stake_percent >= 95., *rpc_percent >= 95.))
+        .unwrap_or((false, false));
 
-    if !feature_activation_allowed && !quiet {
-        if active_stake_by_feature_set.get(&my_feature_set).is_none() {
+    if !stake_allowed && !rpc_allowed && !quiet {
+        if feature_set_stats.get(&my_feature_set).is_none() {
             println!(
                 "{}",
                 style("To activate features the tool and cluster feature sets must match, select a tool version that matches the cluster")
                     .bold());
         } else {
-            println!(
-                "{}",
-                style("To activate features the stake must be >= 95%").bold()
-            );
+            if !stake_allowed {
+                print!(
+                    "\n{}",
+                    style("To activate features the stake must be >= 95%")
+                        .bold()
+                        .red()
+                );
+            }
+            if !rpc_allowed {
+                print!(
+                    "\n{}",
+                    style("To activate features the RPC nodes must be >= 95%")
+                        .bold()
+                        .red()
+                );
+            }
+        }
+        println!(
+            "\n\n{}",
+            style(format!("Tool Feature Set: {}", my_feature_set)).bold()
+        );
+        let feature_set_title = "Feature Set";
+        let stake_percent_title = "Stake";
+        let rpc_percent_title = "RPC";
+        let mut stats_output = Vec::new();
+        let mut max_feature_set_len = feature_set_title.len();
+        let mut max_stake_percent_len = stake_percent_title.len();
+        let mut max_rpc_percent_len = rpc_percent_title.len();
+        for (feature_set, (stake_percent, rpc_percent)) in feature_set_stats.iter() {
+            let me = *feature_set == my_feature_set;
+            let feature_set = if *feature_set == 0 {
+                "unknown".to_string()
+            } else {
+                feature_set.to_string()
+            };
+            let stake_percent = format!("{:.2}%", stake_percent);
+            let rpc_percent = format!("{:.2}%", rpc_percent);
+
+            max_feature_set_len = max_feature_set_len.max(feature_set.len());
+            max_stake_percent_len = max_stake_percent_len.max(stake_percent.len());
+            max_rpc_percent_len = max_rpc_percent_len.max(rpc_percent.len());
+
+            stats_output.push((feature_set, stake_percent, rpc_percent, me));
         }
         println!(
             "{}",
-            style(format!("Tool Feature Set: {}", my_feature_set)).bold()
+            style(format!(
+                "{1:<0$}  {3:<2$}  {5:<4$}",
+                max_feature_set_len,
+                feature_set_title,
+                max_stake_percent_len,
+                stake_percent_title,
+                max_rpc_percent_len,
+                rpc_percent_title,
+            ))
+            .bold(),
         );
-        println!("{}", style("Cluster Feature Sets and Stakes:").bold());
-        for (feature_set, percentage) in active_stake_by_feature_set.iter() {
-            if *feature_set == 0 {
-                println!("  unknown    - {:.2}%", percentage);
-            } else {
-                println!(
-                    "  {:<10} - {:.2}% {}",
-                    feature_set,
-                    percentage,
-                    if *feature_set == my_feature_set {
-                        " <-- me"
-                    } else {
-                        ""
-                    }
-                );
-            }
+        for (feature_set, stake_percent, rpc_percent, me) in stats_output {
+            println!(
+                "{1:>0$}  {3:>2$}  {5:>4$} {6}",
+                max_feature_set_len,
+                feature_set,
+                max_stake_percent_len,
+                stake_percent,
+                max_rpc_percent_len,
+                rpc_percent,
+                if me { "<-- me" } else { "" },
+            );
         }
         println!();
     }
 
-    Ok(feature_activation_allowed)
+    Ok(stake_allowed && rpc_allowed)
 }
 
 fn status_from_account(account: Account) -> Option<CliFeatureStatus> {

cli/src/program.rs

@@ -1146,18 +1146,18 @@ fn get_buffers(
 ) -> Result<CliUpgradeableBuffers, Box<dyn std::error::Error>> {
     let mut filters = vec![RpcFilterType::Memcmp(Memcmp {
         offset: 0,
-        bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 0, 0, 0]).into_string()),
+        bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 0, 0, 0]).into_string()),
         encoding: None,
     })];
     if let Some(authority_pubkey) = authority_pubkey {
         filters.push(RpcFilterType::Memcmp(Memcmp {
             offset: ACCOUNT_TYPE_SIZE,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1]).into_string()),
             encoding: None,
         }));
         filters.push(RpcFilterType::Memcmp(Memcmp {
             offset: ACCOUNT_TYPE_SIZE + OPTION_SIZE,
-            bytes: MemcmpEncodedBytes::Binary(
+            bytes: MemcmpEncodedBytes::Base58(
                 bs58::encode(authority_pubkey.as_ref()).into_string(),
             ),
             encoding: None,
@@ -1199,18 +1199,18 @@ fn get_programs(
 ) -> Result<CliUpgradeablePrograms, Box<dyn std::error::Error>> {
     let mut filters = vec![RpcFilterType::Memcmp(Memcmp {
         offset: 0,
-        bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 0, 0, 0]).into_string()),
+        bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 0, 0, 0]).into_string()),
         encoding: None,
     })];
     if let Some(authority_pubkey) = authority_pubkey {
         filters.push(RpcFilterType::Memcmp(Memcmp {
             offset: ACCOUNT_TYPE_SIZE + SLOT_SIZE,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1]).into_string()),
             encoding: None,
         }));
         filters.push(RpcFilterType::Memcmp(Memcmp {
             offset: ACCOUNT_TYPE_SIZE + SLOT_SIZE + OPTION_SIZE,
-            bytes: MemcmpEncodedBytes::Binary(
+            bytes: MemcmpEncodedBytes::Base58(
                 bs58::encode(authority_pubkey.as_ref()).into_string(),
             ),
             encoding: None,
@@ -1234,7 +1234,7 @@ fn get_programs(
             bytes.extend_from_slice(programdata_address.as_ref());
             let filters = vec![RpcFilterType::Memcmp(Memcmp {
                 offset: 0,
-                bytes: MemcmpEncodedBytes::Binary(bs58::encode(bytes).into_string()),
+                bytes: MemcmpEncodedBytes::Base58(bs58::encode(bytes).into_string()),
                 encoding: None,
             })];
 

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.7.15"
+version = "1.8.1"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -24,14 +24,14 @@ semver = "0.11.0"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.15" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-faucet = { path = "../faucet", version = "=1.7.15" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.15" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.8.1" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-faucet = { path = "../faucet", version = "=1.8.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.8.1" }
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 tungstenite = "0.10.1"
@@ -40,7 +40,7 @@ url = "2.1.1"
 [dev-dependencies]
 assert_matches = "1.3.0"
 jsonrpc-http-server = "18.0.0"
-solana-logger = { path = "../logger", version = "=1.7.15" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/rpc_client.rs

@@ -527,34 +527,50 @@ impl RpcClient {
         Ok(request)
     }
 
-    /// Check the confirmation status of a transaction.
+    /// Submit a transaction and wait for confirmation.
     ///
-    /// Returns `true` if the given transaction succeeded and has been committed
-    /// with the configured [commitment level][cl], which can be retrieved with
-    /// the [`commitment`](RpcClient::commitment) method.
+    /// Once this function returns successfully, the given transaction is
+    /// guaranteed to be processed with the configured [commitment level][cl].
     ///
     /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
     ///
-    /// Note that this method does not wait for a transaction to be confirmed
-    /// — it only checks whether a transaction has been confirmed. To
-    /// submit a transaction and wait for it to confirm, use
-    /// [`send_and_confirm_transaction`][RpcClient::send_and_confirm_transaction].
+    /// After sending the transaction, this method polls in a loop for the
+    /// status of the transaction until it has ben confirmed.
     ///
-    /// _This method returns `false` if the transaction failed, even if it has
-    /// been confirmed._
+    /// # Errors
+    ///
+    /// If the transaction is not signed then an error with kind [`RpcError`] is
+    /// returned, containing an [`RpcResponseError`] with `code` set to
+    /// [`JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE`].
+    ///
+    /// If the preflight transaction simulation fails then an error with kind
+    /// [`RpcError`] is returned, containing an [`RpcResponseError`] with `code`
+    /// set to [`JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE`].
+    ///
+    /// If the receiving node is unhealthy, e.g. it is not fully synced to
+    /// the cluster, then an error with kind [`RpcError`] is returned,
+    /// containing an [`RpcResponseError`] with `code` set to
+    /// [`JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY`].
+    ///
+    /// [`RpcResponseError`]: RpcError::RpcResponseError
+    /// [`JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE
+    /// [`JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE
+    /// [`JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY
     ///
     /// # RPC Reference
     ///
-    /// This method is built on the [`getSignatureStatuses`] RPC method.
+    /// This method is built on the [`sendTransaction`] RPC method, and the
+    /// [`getLatestBlockhash`] RPC method.
     ///
-    /// [`getSignatureStatuses`]: https://docs.solana.com/developing/clients/jsonrpc-api#getsignaturestatuses
+    /// [`sendTransaction`]: https://docs.solana.com/developing/clients/jsonrpc-api#sendtransaction
+    /// [`getLatestBlockhash`]: https://docs.solana.com/developing/clients/jsonrpc-api#getlatestblockhash
     ///
     /// # Examples
     ///
     /// ```
     /// # use solana_client::{
-    /// #     client_error::ClientError,
     /// #     rpc_client::RpcClient,
+    /// #     client_error::ClientError,
     /// # };
     /// # use solana_sdk::{
     /// #     signature::Signer,
@@ -563,97 +579,110 @@ impl RpcClient {
     /// #     system_transaction,
     /// # };
     /// # let rpc_client = RpcClient::new_mock("succeeds".to_string());
-    /// // Transfer lamports from Alice to Bob and wait for confirmation
     /// # let alice = Keypair::new();
     /// # let bob = Keypair::new();
     /// # let lamports = 50;
-    /// let (recent_blockhash, _) = rpc_client.get_recent_blockhash()?;
+    /// # let (recent_blockhash, _) = rpc_client.get_recent_blockhash()?;
     /// let tx = system_transaction::transfer(&alice, &bob.pubkey(), lamports, recent_blockhash);
-    /// let signature = rpc_client.send_transaction(&tx)?;
-    ///
-    /// loop {
-    ///     let confirmed = rpc_client.confirm_transaction(&signature)?;
-    ///     if confirmed {
-    ///         break;
-    ///     }
-    /// }
+    /// let signature = rpc_client.send_and_confirm_transaction(&tx)?;
     /// # Ok::<(), ClientError>(())
     /// ```
-    pub fn confirm_transaction(&self, signature: &Signature) -> ClientResult<bool> {
-        Ok(self
-            .confirm_transaction_with_commitment(signature, self.commitment())?
-            .value)
+    pub fn send_and_confirm_transaction(
+        &self,
+        transaction: &Transaction,
+    ) -> ClientResult<Signature> {
+        const SEND_RETRIES: usize = 1;
+        const GET_STATUS_RETRIES: usize = usize::MAX;
+
+        'sending: for _ in 0..SEND_RETRIES {
+            let signature = self.send_transaction(transaction)?;
+
+            let recent_blockhash = if uses_durable_nonce(transaction).is_some() {
+                let (recent_blockhash, ..) = self
+                    .get_recent_blockhash_with_commitment(CommitmentConfig::processed())?
+                    .value;
+                recent_blockhash
+            } else {
+                transaction.message.recent_blockhash
+            };
+
+            for status_retry in 0..GET_STATUS_RETRIES {
+                match self.get_signature_status(&signature)? {
+                    Some(Ok(_)) => return Ok(signature),
+                    Some(Err(e)) => return Err(e.into()),
+                    None => {
+                        let fee_calculator = self
+                            .get_fee_calculator_for_blockhash_with_commitment(
+                                &recent_blockhash,
+                                CommitmentConfig::processed(),
+                            )?
+                            .value;
+                        if fee_calculator.is_none() {
+                            // Block hash is not found for some reason
+                            break 'sending;
+                        } else if cfg!(not(test))
+                            // Ignore sleep at last step.
+                            && status_retry < GET_STATUS_RETRIES
+                        {
+                            // Retry twice a second
+                            sleep(Duration::from_millis(500));
+                            continue;
+                        }
+                    }
+                }
+            }
+        }
+
+        Err(RpcError::ForUser(
+            "unable to confirm transaction. \
+             This can happen in situations such as transaction expiration \
+             and insufficient fee-payer funds"
+                .to_string(),
+        )
+        .into())
     }
 
-    /// Check the confirmation status of a transaction.
-    ///
-    /// Returns an [`RpcResult`] with value `true` if the given transaction
-    /// succeeded and has been committed with the given [commitment level][cl].
-    ///
-    /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
-    ///
-    /// Note that this method does not wait for a transaction to be confirmed
-    /// &mdash; it only checks whether a transaction has been confirmed. To
-    /// submit a transaction and wait for it to confirm, use
-    /// [`send_and_confirm_transaction`][RpcClient::send_and_confirm_transaction].
-    ///
-    /// _This method returns an [`RpcResult`] with value `false` if the
-    /// transaction failed, even if it has been confirmed._
-    ///
-    /// # RPC Reference
-    ///
-    /// This method is built on the [`getSignatureStatuses`] RPC method.
-    ///
-    /// [`getSignatureStatuses`]: https://docs.solana.com/developing/clients/jsonrpc-api#getsignaturestatuses
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// # use solana_client::{
-    /// #     client_error::ClientError,
-    /// #     rpc_client::RpcClient,
-    /// # };
-    /// # use solana_sdk::{
-    /// #     commitment_config::CommitmentConfig,
-    /// #     signature::Signer,
-    /// #     signature::Signature,
-    /// #     signer::keypair::Keypair,
-    /// #     system_transaction,
-    /// # };
-    /// # use std::time::Duration;
-    /// # let rpc_client = RpcClient::new_mock("succeeds".to_string());
-    /// // Transfer lamports from Alice to Bob and wait for confirmation
-    /// # let alice = Keypair::new();
-    /// # let bob = Keypair::new();
-    /// # let lamports = 50;
-    /// let (recent_blockhash, _) = rpc_client.get_recent_blockhash()?;
-    /// let tx = system_transaction::transfer(&alice, &bob.pubkey(), lamports, recent_blockhash);
-    /// let signature = rpc_client.send_transaction(&tx)?;
-    ///
-    /// loop {
-    ///     let commitment_config = CommitmentConfig::processed();
-    ///     let confirmed = rpc_client.confirm_transaction_with_commitment(&signature, commitment_config)?;
-    ///     if confirmed.value {
-    ///         break;
-    ///     }
-    /// }
-    /// # Ok::<(), ClientError>(())
-    /// ```
-    pub fn confirm_transaction_with_commitment(
+    pub fn send_and_confirm_transaction_with_spinner(
         &self,
-        signature: &Signature,
-        commitment_config: CommitmentConfig,
-    ) -> RpcResult<bool> {
-        let Response { context, value } = self.get_signature_statuses(&[*signature])?;
+        transaction: &Transaction,
+    ) -> ClientResult<Signature> {
+        self.send_and_confirm_transaction_with_spinner_and_commitment(
+            transaction,
+            self.commitment(),
+        )
+    }
 
-        Ok(Response {
-            context,
-            value: value[0]
-                .as_ref()
-                .filter(|result| result.satisfies_commitment(commitment_config))
-                .map(|result| result.status.is_ok())
-                .unwrap_or_default(),
-        })
+    pub fn send_and_confirm_transaction_with_spinner_and_commitment(
+        &self,
+        transaction: &Transaction,
+        commitment: CommitmentConfig,
+    ) -> ClientResult<Signature> {
+        self.send_and_confirm_transaction_with_spinner_and_config(
+            transaction,
+            commitment,
+            RpcSendTransactionConfig {
+                preflight_commitment: Some(commitment.commitment),
+                ..RpcSendTransactionConfig::default()
+            },
+        )
+    }
+
+    pub fn send_and_confirm_transaction_with_spinner_and_config(
+        &self,
+        transaction: &Transaction,
+        commitment: CommitmentConfig,
+        config: RpcSendTransactionConfig,
+    ) -> ClientResult<Signature> {
+        let recent_blockhash = if uses_durable_nonce(transaction).is_some() {
+            self.get_recent_blockhash_with_commitment(CommitmentConfig::processed())?
+                .value
+                .0
+        } else {
+            transaction.message.recent_blockhash
+        };
+        let signature = self.send_transaction_with_config(transaction, config)?;
+        self.confirm_transaction_with_spinner(&signature, &recent_blockhash, commitment)?;
+        Ok(signature)
     }
 
     /// Submits a signed transaction to the network.
@@ -737,14 +766,6 @@ impl RpcClient {
         )
     }
 
-    fn default_cluster_transaction_encoding(&self) -> Result<UiTransactionEncoding, RpcError> {
-        if self.get_node_version()? < semver::Version::new(1, 3, 16) {
-            Ok(UiTransactionEncoding::Base58)
-        } else {
-            Ok(UiTransactionEncoding::Base64)
-        }
-    }
-
     /// Submits a signed transaction to the network.
     ///
     /// Before a transaction is processed, the receiving node runs a "preflight
@@ -890,6 +911,251 @@ impl RpcClient {
         }
     }
 
+    pub fn send<T>(&self, request: RpcRequest, params: Value) -> ClientResult<T>
+    where
+        T: serde::de::DeserializeOwned,
+    {
+        assert!(params.is_array() || params.is_null());
+
+        let response = self
+            .sender
+            .send(request, params)
+            .map_err(|err| err.into_with_request(request))?;
+        serde_json::from_value(response)
+            .map_err(|err| ClientError::new_with_request(err.into(), request))
+    }
+
+    /// Check the confirmation status of a transaction.
+    ///
+    /// Returns `true` if the given transaction succeeded and has been committed
+    /// with the configured [commitment level][cl], which can be retrieved with
+    /// the [`commitment`](RpcClient::commitment) method.
+    ///
+    /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
+    ///
+    /// Note that this method does not wait for a transaction to be confirmed
+    /// &mdash; it only checks whether a transaction has been confirmed. To
+    /// submit a transaction and wait for it to confirm, use
+    /// [`send_and_confirm_transaction`][RpcClient::send_and_confirm_transaction].
+    ///
+    /// _This method returns `false` if the transaction failed, even if it has
+    /// been confirmed._
+    ///
+    /// # RPC Reference
+    ///
+    /// This method is built on the [`getSignatureStatuses`] RPC method.
+    ///
+    /// [`getSignatureStatuses`]: https://docs.solana.com/developing/clients/jsonrpc-api#getsignaturestatuses
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use solana_client::{
+    /// #     client_error::ClientError,
+    /// #     rpc_client::RpcClient,
+    /// # };
+    /// # use solana_sdk::{
+    /// #     signature::Signer,
+    /// #     signature::Signature,
+    /// #     signer::keypair::Keypair,
+    /// #     system_transaction,
+    /// # };
+    /// # let rpc_client = RpcClient::new_mock("succeeds".to_string());
+    /// // Transfer lamports from Alice to Bob and wait for confirmation
+    /// # let alice = Keypair::new();
+    /// # let bob = Keypair::new();
+    /// # let lamports = 50;
+    /// let (recent_blockhash, _) = rpc_client.get_recent_blockhash()?;
+    /// let tx = system_transaction::transfer(&alice, &bob.pubkey(), lamports, recent_blockhash);
+    /// let signature = rpc_client.send_transaction(&tx)?;
+    ///
+    /// loop {
+    ///     let confirmed = rpc_client.confirm_transaction(&signature)?;
+    ///     if confirmed {
+    ///         break;
+    ///     }
+    /// }
+    /// # Ok::<(), ClientError>(())
+    /// ```
+    pub fn confirm_transaction(&self, signature: &Signature) -> ClientResult<bool> {
+        Ok(self
+            .confirm_transaction_with_commitment(signature, self.commitment())?
+            .value)
+    }
+
+    /// Check the confirmation status of a transaction.
+    ///
+    /// Returns an [`RpcResult`] with value `true` if the given transaction
+    /// succeeded and has been committed with the given [commitment level][cl].
+    ///
+    /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
+    ///
+    /// Note that this method does not wait for a transaction to be confirmed
+    /// &mdash; it only checks whether a transaction has been confirmed. To
+    /// submit a transaction and wait for it to confirm, use
+    /// [`send_and_confirm_transaction`][RpcClient::send_and_confirm_transaction].
+    ///
+    /// _This method returns an [`RpcResult`] with value `false` if the
+    /// transaction failed, even if it has been confirmed._
+    ///
+    /// # RPC Reference
+    ///
+    /// This method is built on the [`getSignatureStatuses`] RPC method.
+    ///
+    /// [`getSignatureStatuses`]: https://docs.solana.com/developing/clients/jsonrpc-api#getsignaturestatuses
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use solana_client::{
+    /// #     client_error::ClientError,
+    /// #     rpc_client::RpcClient,
+    /// # };
+    /// # use solana_sdk::{
+    /// #     commitment_config::CommitmentConfig,
+    /// #     signature::Signer,
+    /// #     signature::Signature,
+    /// #     signer::keypair::Keypair,
+    /// #     system_transaction,
+    /// # };
+    /// # use std::time::Duration;
+    /// # let rpc_client = RpcClient::new_mock("succeeds".to_string());
+    /// // Transfer lamports from Alice to Bob and wait for confirmation
+    /// # let alice = Keypair::new();
+    /// # let bob = Keypair::new();
+    /// # let lamports = 50;
+    /// let (recent_blockhash, _) = rpc_client.get_recent_blockhash()?;
+    /// let tx = system_transaction::transfer(&alice, &bob.pubkey(), lamports, recent_blockhash);
+    /// let signature = rpc_client.send_transaction(&tx)?;
+    ///
+    /// loop {
+    ///     let commitment_config = CommitmentConfig::processed();
+    ///     let confirmed = rpc_client.confirm_transaction_with_commitment(&signature, commitment_config)?;
+    ///     if confirmed.value {
+    ///         break;
+    ///     }
+    /// }
+    /// # Ok::<(), ClientError>(())
+    /// ```
+    pub fn confirm_transaction_with_commitment(
+        &self,
+        signature: &Signature,
+        commitment_config: CommitmentConfig,
+    ) -> RpcResult<bool> {
+        let Response { context, value } = self.get_signature_statuses(&[*signature])?;
+
+        Ok(Response {
+            context,
+            value: value[0]
+                .as_ref()
+                .filter(|result| result.satisfies_commitment(commitment_config))
+                .map(|result| result.status.is_ok())
+                .unwrap_or_default(),
+        })
+    }
+
+    pub fn confirm_transaction_with_spinner(
+        &self,
+        signature: &Signature,
+        recent_blockhash: &Hash,
+        commitment: CommitmentConfig,
+    ) -> ClientResult<()> {
+        let desired_confirmations = if commitment.is_finalized() {
+            MAX_LOCKOUT_HISTORY + 1
+        } else {
+            1
+        };
+        let mut confirmations = 0;
+
+        let progress_bar = new_spinner_progress_bar();
+
+        progress_bar.set_message(&format!(
+            "[{}/{}] Finalizing transaction {}",
+            confirmations, desired_confirmations, signature,
+        ));
+
+        let now = Instant::now();
+        let confirm_transaction_initial_timeout = self
+            .config
+            .confirm_transaction_initial_timeout
+            .unwrap_or_default();
+        let (signature, status) = loop {
+            // Get recent commitment in order to count confirmations for successful transactions
+            let status = self
+                .get_signature_status_with_commitment(signature, CommitmentConfig::processed())?;
+            if status.is_none() {
+                let blockhash_not_found = self
+                    .get_fee_calculator_for_blockhash_with_commitment(
+                        recent_blockhash,
+                        CommitmentConfig::processed(),
+                    )?
+                    .value
+                    .is_none();
+                if blockhash_not_found && now.elapsed() >= confirm_transaction_initial_timeout {
+                    break (signature, status);
+                }
+            } else {
+                break (signature, status);
+            }
+
+            if cfg!(not(test)) {
+                sleep(Duration::from_millis(500));
+            }
+        };
+        if let Some(result) = status {
+            if let Err(err) = result {
+                return Err(err.into());
+            }
+        } else {
+            return Err(RpcError::ForUser(
+                "unable to confirm transaction. \
+                                      This can happen in situations such as transaction expiration \
+                                      and insufficient fee-payer funds"
+                    .to_string(),
+            )
+            .into());
+        }
+        let now = Instant::now();
+        loop {
+            // Return when specified commitment is reached
+            // Failed transactions have already been eliminated, `is_some` check is sufficient
+            if self
+                .get_signature_status_with_commitment(signature, commitment)?
+                .is_some()
+            {
+                progress_bar.set_message("Transaction confirmed");
+                progress_bar.finish_and_clear();
+                return Ok(());
+            }
+
+            progress_bar.set_message(&format!(
+                "[{}/{}] Finalizing transaction {}",
+                min(confirmations + 1, desired_confirmations),
+                desired_confirmations,
+                signature,
+            ));
+            sleep(Duration::from_millis(500));
+            confirmations = self
+                .get_num_blocks_since_signature_confirmation(signature)
+                .unwrap_or(confirmations);
+            if now.elapsed().as_secs() >= MAX_HASH_AGE_IN_SECONDS as u64 {
+                return Err(
+                    RpcError::ForUser("transaction not finalized. \
+                                      This can happen when a transaction lands in an abandoned fork. \
+                                      Please retry.".to_string()).into(),
+                );
+            }
+        }
+    }
+
+    fn default_cluster_transaction_encoding(&self) -> Result<UiTransactionEncoding, RpcError> {
+        if self.get_node_version()? < semver::Version::new(1, 3, 16) {
+            Ok(UiTransactionEncoding::Base58)
+        } else {
+            Ok(UiTransactionEncoding::Base64)
+        }
+    }
+
     /// Simulates sending a transaction.
     ///
     /// If the transaction fails, then the [`err`] field of the returned
@@ -3223,121 +3489,6 @@ impl RpcClient {
         self.send(RpcRequest::MinimumLedgerSlot, Value::Null)
     }
 
-    /// Submit a transaction and wait for confirmation.
-    ///
-    /// Once this function returns successfully, the given transaction is
-    /// guaranteed to be processed with the configured [commitment level][cl].
-    ///
-    /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
-    ///
-    /// After sending the transaction, this method polls in a loop for the
-    /// status of the transaction until it has ben confirmed.
-    ///
-    /// # Errors
-    ///
-    /// If the transaction is not signed then an error with kind [`RpcError`] is
-    /// returned, containing an [`RpcResponseError`] with `code` set to
-    /// [`JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE`].
-    ///
-    /// If the preflight transaction simulation fails then an error with kind
-    /// [`RpcError`] is returned, containing an [`RpcResponseError`] with `code`
-    /// set to [`JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE`].
-    ///
-    /// If the receiving node is unhealthy, e.g. it is not fully synced to
-    /// the cluster, then an error with kind [`RpcError`] is returned,
-    /// containing an [`RpcResponseError`] with `code` set to
-    /// [`JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY`].
-    ///
-    /// [`RpcResponseError`]: RpcError::RpcResponseError
-    /// [`JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE
-    /// [`JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE
-    /// [`JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY
-    ///
-    /// # RPC Reference
-    ///
-    /// This method is built on the [`sendTransaction`] RPC method, and the
-    /// [`getLatestBlockhash`] RPC method.
-    ///
-    /// [`sendTransaction`]: https://docs.solana.com/developing/clients/jsonrpc-api#sendtransaction
-    /// [`getLatestBlockhash`]: https://docs.solana.com/developing/clients/jsonrpc-api#getlatestblockhash
-    ///
-    /// # Examples
-    ///
-    /// ```
-    /// # use solana_client::{
-    /// #     rpc_client::RpcClient,
-    /// #     client_error::ClientError,
-    /// # };
-    /// # use solana_sdk::{
-    /// #     signature::Signer,
-    /// #     signature::Signature,
-    /// #     signer::keypair::Keypair,
-    /// #     system_transaction,
-    /// # };
-    /// # let rpc_client = RpcClient::new_mock("succeeds".to_string());
-    /// # let alice = Keypair::new();
-    /// # let bob = Keypair::new();
-    /// # let lamports = 50;
-    /// # let recent_blockhash = rpc_client.get_recent_blockhash()?.0;
-    /// let tx = system_transaction::transfer(&alice, &bob.pubkey(), lamports, recent_blockhash);
-    /// let signature = rpc_client.send_and_confirm_transaction(&tx)?;
-    /// # Ok::<(), ClientError>(())
-    /// ```
-    pub fn send_and_confirm_transaction(
-        &self,
-        transaction: &Transaction,
-    ) -> ClientResult<Signature> {
-        const SEND_RETRIES: usize = 1;
-        const GET_STATUS_RETRIES: usize = usize::MAX;
-
-        'sending: for _ in 0..SEND_RETRIES {
-            let signature = self.send_transaction(transaction)?;
-
-            let recent_blockhash = if uses_durable_nonce(transaction).is_some() {
-                let (recent_blockhash, ..) = self
-                    .get_recent_blockhash_with_commitment(CommitmentConfig::processed())?
-                    .value;
-                recent_blockhash
-            } else {
-                transaction.message.recent_blockhash
-            };
-
-            for status_retry in 0..GET_STATUS_RETRIES {
-                match self.get_signature_status(&signature)? {
-                    Some(Ok(_)) => return Ok(signature),
-                    Some(Err(e)) => return Err(e.into()),
-                    None => {
-                        let fee_calculator = self
-                            .get_fee_calculator_for_blockhash_with_commitment(
-                                &recent_blockhash,
-                                CommitmentConfig::processed(),
-                            )?
-                            .value;
-                        if fee_calculator.is_none() {
-                            // Block hash is not found by some reason
-                            break 'sending;
-                        } else if cfg!(not(test))
-                            // Ignore sleep at last step.
-                            && status_retry < GET_STATUS_RETRIES
-                        {
-                            // Retry twice a second
-                            sleep(Duration::from_millis(500));
-                            continue;
-                        }
-                    }
-                }
-            }
-        }
-
-        Err(RpcError::ForUser(
-            "unable to confirm transaction. \
-             This can happen in situations such as transaction expiration \
-             and insufficient fee-payer funds"
-                .to_string(),
-        )
-        .into())
-    }
-
     /// Returns all information associated with the account of the provided pubkey.
     ///
     /// This method uses the configured [commitment level][cl].
@@ -4541,157 +4692,6 @@ impl RpcClient {
         Ok(confirmations)
     }
 
-    pub fn send_and_confirm_transaction_with_spinner(
-        &self,
-        transaction: &Transaction,
-    ) -> ClientResult<Signature> {
-        self.send_and_confirm_transaction_with_spinner_and_commitment(
-            transaction,
-            self.commitment(),
-        )
-    }
-
-    pub fn send_and_confirm_transaction_with_spinner_and_commitment(
-        &self,
-        transaction: &Transaction,
-        commitment: CommitmentConfig,
-    ) -> ClientResult<Signature> {
-        self.send_and_confirm_transaction_with_spinner_and_config(
-            transaction,
-            commitment,
-            RpcSendTransactionConfig {
-                preflight_commitment: Some(commitment.commitment),
-                ..RpcSendTransactionConfig::default()
-            },
-        )
-    }
-
-    pub fn send_and_confirm_transaction_with_spinner_and_config(
-        &self,
-        transaction: &Transaction,
-        commitment: CommitmentConfig,
-        config: RpcSendTransactionConfig,
-    ) -> ClientResult<Signature> {
-        let recent_blockhash = if uses_durable_nonce(transaction).is_some() {
-            self.get_recent_blockhash_with_commitment(CommitmentConfig::processed())?
-                .value
-                .0
-        } else {
-            transaction.message.recent_blockhash
-        };
-        let signature = self.send_transaction_with_config(transaction, config)?;
-        self.confirm_transaction_with_spinner(&signature, &recent_blockhash, commitment)?;
-        Ok(signature)
-    }
-
-    pub fn confirm_transaction_with_spinner(
-        &self,
-        signature: &Signature,
-        recent_blockhash: &Hash,
-        commitment: CommitmentConfig,
-    ) -> ClientResult<()> {
-        let desired_confirmations = if commitment.is_finalized() {
-            MAX_LOCKOUT_HISTORY + 1
-        } else {
-            1
-        };
-        let mut confirmations = 0;
-
-        let progress_bar = new_spinner_progress_bar();
-
-        progress_bar.set_message(&format!(
-            "[{}/{}] Finalizing transaction {}",
-            confirmations, desired_confirmations, signature,
-        ));
-
-        let now = Instant::now();
-        let confirm_transaction_initial_timeout = self
-            .config
-            .confirm_transaction_initial_timeout
-            .unwrap_or_default();
-        let (signature, status) = loop {
-            // Get recent commitment in order to count confirmations for successful transactions
-            let status = self
-                .get_signature_status_with_commitment(signature, CommitmentConfig::processed())?;
-            if status.is_none() {
-                let blockhash_not_found = self
-                    .get_fee_calculator_for_blockhash_with_commitment(
-                        recent_blockhash,
-                        CommitmentConfig::processed(),
-                    )?
-                    .value
-                    .is_none();
-                if blockhash_not_found && now.elapsed() >= confirm_transaction_initial_timeout {
-                    break (signature, status);
-                }
-            } else {
-                break (signature, status);
-            }
-
-            if cfg!(not(test)) {
-                sleep(Duration::from_millis(500));
-            }
-        };
-        if let Some(result) = status {
-            if let Err(err) = result {
-                return Err(err.into());
-            }
-        } else {
-            return Err(RpcError::ForUser(
-                "unable to confirm transaction. \
-                                      This can happen in situations such as transaction expiration \
-                                      and insufficient fee-payer funds"
-                    .to_string(),
-            )
-            .into());
-        }
-        let now = Instant::now();
-        loop {
-            // Return when specified commitment is reached
-            // Failed transactions have already been eliminated, `is_some` check is sufficient
-            if self
-                .get_signature_status_with_commitment(signature, commitment)?
-                .is_some()
-            {
-                progress_bar.set_message("Transaction confirmed");
-                progress_bar.finish_and_clear();
-                return Ok(());
-            }
-
-            progress_bar.set_message(&format!(
-                "[{}/{}] Finalizing transaction {}",
-                min(confirmations + 1, desired_confirmations),
-                desired_confirmations,
-                signature,
-            ));
-            sleep(Duration::from_millis(500));
-            confirmations = self
-                .get_num_blocks_since_signature_confirmation(signature)
-                .unwrap_or(confirmations);
-            if now.elapsed().as_secs() >= MAX_HASH_AGE_IN_SECONDS as u64 {
-                return Err(
-                    RpcError::ForUser("transaction not finalized. \
-                                      This can happen when a transaction lands in an abandoned fork. \
-                                      Please retry.".to_string()).into(),
-                );
-            }
-        }
-    }
-
-    pub fn send<T>(&self, request: RpcRequest, params: Value) -> ClientResult<T>
-    where
-        T: serde::de::DeserializeOwned,
-    {
-        assert!(params.is_array() || params.is_null());
-
-        let response = self
-            .sender
-            .send(request, params)
-            .map_err(|err| err.into_with_request(request))?;
-        serde_json::from_value(response)
-            .map_err(|err| ClientError::new_with_request(err.into(), request))
-    }
-
     pub fn get_transport_stats(&self) -> RpcTransportStats {
         self.sender.get_transport_stats()
     }

client/src/rpc_filter.rs

@@ -1,4 +1,9 @@
-use thiserror::Error;
+#![allow(deprecated)]
+use {std::borrow::Cow, thiserror::Error};
+
+const MAX_DATA_SIZE: usize = 128;
+const MAX_DATA_BASE58_SIZE: usize = 175;
+const MAX_DATA_BASE64_SIZE: usize = 172;
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -15,15 +20,50 @@ impl RpcFilterType {
                 let encoding = compare.encoding.as_ref().unwrap_or(&MemcmpEncoding::Binary);
                 match encoding {
                     MemcmpEncoding::Binary => {
-                        let MemcmpEncodedBytes::Binary(bytes) = &compare.bytes;
-
-                        if bytes.len() > 128 {
-                            Err(RpcFilterError::Base58DataTooLarge)
-                        } else {
-                            bs58::decode(&bytes)
-                                .into_vec()
-                                .map(|_| ())
-                                .map_err(|e| e.into())
+                        use MemcmpEncodedBytes::*;
+                        match &compare.bytes {
+                            Binary(bytes) if bytes.len() > MAX_DATA_BASE58_SIZE => {
+                                Err(RpcFilterError::Base58DataTooLarge)
+                            }
+                            Base58(bytes) if bytes.len() > MAX_DATA_BASE58_SIZE => {
+                                Err(RpcFilterError::DataTooLarge)
+                            }
+                            Base64(bytes) if bytes.len() > MAX_DATA_BASE64_SIZE => {
+                                Err(RpcFilterError::DataTooLarge)
+                            }
+                            Bytes(bytes) if bytes.len() > MAX_DATA_SIZE => {
+                                Err(RpcFilterError::DataTooLarge)
+                            }
+                            _ => Ok(()),
+                        }?;
+                        match &compare.bytes {
+                            Binary(bytes) => {
+                                let bytes = bs58::decode(&bytes)
+                                    .into_vec()
+                                    .map_err(RpcFilterError::DecodeError)?;
+                                if bytes.len() > MAX_DATA_SIZE {
+                                    Err(RpcFilterError::Base58DataTooLarge)
+                                } else {
+                                    Ok(())
+                                }
+                            }
+                            Base58(bytes) => {
+                                let bytes = bs58::decode(&bytes).into_vec()?;
+                                if bytes.len() > MAX_DATA_SIZE {
+                                    Err(RpcFilterError::DataTooLarge)
+                                } else {
+                                    Ok(())
+                                }
+                            }
+                            Base64(bytes) => {
+                                let bytes = base64::decode(&bytes)?;
+                                if bytes.len() > MAX_DATA_SIZE {
+                                    Err(RpcFilterError::DataTooLarge)
+                                } else {
+                                    Ok(())
+                                }
+                            }
+                            Bytes(_) => Ok(()),
                         }
                     }
                 }
@@ -34,10 +74,24 @@ impl RpcFilterType {
 
 #[derive(Error, PartialEq, Debug)]
 pub enum RpcFilterError {
-    #[error("bs58 decode error")]
-    DecodeError(#[from] bs58::decode::Error),
+    #[error("encoded binary data should be less than 129 bytes")]
+    DataTooLarge,
+    #[deprecated(
+        since = "1.9.0",
+        note = "Error for MemcmpEncodedBytes::Binary which is deprecated"
+    )]
     #[error("encoded binary (base 58) data should be less than 129 bytes")]
     Base58DataTooLarge,
+    #[deprecated(
+        since = "1.9.0",
+        note = "Error for MemcmpEncodedBytes::Binary which is deprecated"
+    )]
+    #[error("bs58 decode error")]
+    DecodeError(bs58::decode::Error),
+    #[error("base58 decode error")]
+    Base58DecodeError(#[from] bs58::decode::Error),
+    #[error("base64 decode error")]
+    Base64DecodeError(#[from] base64::DecodeError),
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -49,7 +103,14 @@ pub enum MemcmpEncoding {
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase", untagged)]
 pub enum MemcmpEncodedBytes {
+    #[deprecated(
+        since = "1.9.0",
+        note = "Please use MemcmpEncodedBytes::Base58 instead"
+    )]
     Binary(String),
+    Base58(String),
+    Base64(String),
+    Bytes(Vec<u8>),
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -63,14 +124,18 @@ pub struct Memcmp {
 }
 
 impl Memcmp {
-    pub fn bytes_match(&self, data: &[u8]) -> bool {
+    pub fn bytes(&self) -> Option<Cow<Vec<u8>>> {
+        use MemcmpEncodedBytes::*;
         match &self.bytes {
-            MemcmpEncodedBytes::Binary(bytes) => {
-                let bytes = bs58::decode(bytes).into_vec();
-                if bytes.is_err() {
-                    return false;
-                }
-                let bytes = bytes.unwrap();
+            Binary(bytes) | Base58(bytes) => bs58::decode(bytes).into_vec().ok().map(Cow::Owned),
+            Base64(bytes) => base64::decode(bytes).ok().map(Cow::Owned),
+            Bytes(bytes) => Some(Cow::Borrowed(bytes)),
+        }
+    }
+
+    pub fn bytes_match(&self, data: &[u8]) -> bool {
+        match self.bytes() {
+            Some(bytes) => {
                 if self.offset > data.len() {
                     return false;
                 }
@@ -79,6 +144,7 @@ impl Memcmp {
                 }
                 data[self.offset..self.offset + bytes.len()] == bytes[..]
             }
+            None => false,
         }
     }
 }
@@ -87,6 +153,15 @@ impl Memcmp {
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_worst_case_encoded_tx_goldens() {
+        let ff_data = vec![0xffu8; MAX_DATA_SIZE];
+        let data58 = bs58::encode(&ff_data).into_string();
+        assert_eq!(data58.len(), MAX_DATA_BASE58_SIZE);
+        let data64 = base64::encode(&ff_data);
+        assert_eq!(data64.len(), MAX_DATA_BASE64_SIZE);
+    }
+
     #[test]
     fn test_bytes_match() {
         let data = vec![1, 2, 3, 4, 5];
@@ -94,7 +169,7 @@ mod tests {
         // Exact match of data succeeds
         assert!(Memcmp {
             offset: 0,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()),
             encoding: None,
         }
         .bytes_match(&data));
@@ -102,7 +177,7 @@ mod tests {
         // Partial match of data succeeds
         assert!(Memcmp {
             offset: 0,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 2]).into_string()),
             encoding: None,
         }
         .bytes_match(&data));
@@ -110,7 +185,7 @@ mod tests {
         // Offset partial match of data succeeds
         assert!(Memcmp {
             offset: 2,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 4]).into_string()),
             encoding: None,
         }
         .bytes_match(&data));
@@ -118,7 +193,7 @@ mod tests {
         // Incorrect partial match of data fails
         assert!(!Memcmp {
             offset: 0,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![2]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![2]).into_string()),
             encoding: None,
         }
         .bytes_match(&data));
@@ -126,7 +201,7 @@ mod tests {
         // Bytes overrun data fails
         assert!(!Memcmp {
             offset: 2,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4, 5, 6]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 4, 5, 6]).into_string()),
             encoding: None,
         }
         .bytes_match(&data));
@@ -134,7 +209,7 @@ mod tests {
         // Offset outside data fails
         assert!(!Memcmp {
             offset: 6,
-            bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![5]).into_string()),
+            bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![5]).into_string()),
             encoding: None,
         }
         .bytes_match(&data));
@@ -142,7 +217,7 @@ mod tests {
         // Invalid base-58 fails
         assert!(!Memcmp {
             offset: 0,
-            bytes: MemcmpEncodedBytes::Binary("III".to_string()),
+            bytes: MemcmpEncodedBytes::Base58("III".to_string()),
             encoding: None,
         }
         .bytes_match(&data));
@@ -157,7 +232,7 @@ mod tests {
         assert_eq!(
             RpcFilterType::Memcmp(Memcmp {
                 offset: 0,
-                bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()),
+                bytes: MemcmpEncodedBytes::Base58(base58_bytes.to_string()),
                 encoding: None,
             })
             .verify(),
@@ -172,11 +247,11 @@ mod tests {
         assert_eq!(
             RpcFilterType::Memcmp(Memcmp {
                 offset: 0,
-                bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()),
+                bytes: MemcmpEncodedBytes::Base58(base58_bytes.to_string()),
                 encoding: None,
             })
             .verify(),
-            Err(RpcFilterError::Base58DataTooLarge)
+            Err(RpcFilterError::DataTooLarge)
         );
     }
 }

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.15"
+version = "1.8.1"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-core"
 readme = "../README.md"
@@ -27,13 +27,14 @@ ed25519-dalek = "=1.0.1"
 fs_extra = "1.2.0"
 flate2 = "1.0"
 indexmap = { version = "1.5", features = ["rayon"] }
-itertools = "0.9.0"
 libc = "0.2.81"
-log = "0.4.11"
-lru = "0.6.1"
 miow = "0.2.2"
 net2 = "0.2.37"
 num-traits = "0.2"
+histogram = "0.6.9"
+itertools = "0.10.1"
+log = "0.4.14"
+lru = "0.6.6"
 rand = "0.7.0"
 rand_chacha = "0.2.2"
 rand_core = "0.6.2"
@@ -43,33 +44,34 @@ retain_mut = "0.1.2"
 serde = "1.0.122"
 serde_bytes = "0.11"
 serde_derive = "1.0.103"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.15" }
-solana-banks-server = { path = "../banks-server", version = "=1.7.15" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-client = { path = "../client", version = "=1.7.15" }
-solana-gossip = { path = "../gossip", version = "=1.7.15" }
-solana-ledger = { path = "../ledger", version = "=1.7.15" }
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-merkle-tree = { path = "../merkle-tree", version = "=1.7.15" }
-solana-metrics = { path = "../metrics", version = "=1.7.15" }
-solana-measure = { path = "../measure", version = "=1.7.15" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.15" }
-solana-perf = { path = "../perf", version = "=1.7.15" }
-solana-poh = { path = "../poh", version = "=1.7.15" }
-solana-program-test = { path = "../program-test", version = "=1.7.15" }
-solana-rpc = { path = "../rpc", version = "=1.7.15" }
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-frozen-abi = { path = "../frozen-abi", version = "=1.7.15" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.7.15" }
-solana-streamer = { path = "../streamer", version = "=1.7.15" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.15" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.8.1" }
+solana-accountsdb-plugin-manager = { path = "../accountsdb-plugin-manager", version = "=1.8.1" }
+solana-banks-server = { path = "../banks-server", version = "=1.8.1" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-client = { path = "../client", version = "=1.8.1" }
+solana-gossip = { path = "../gossip", version = "=1.8.1" }
+solana-ledger = { path = "../ledger", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-merkle-tree = { path = "../merkle-tree", version = "=1.8.1" }
+solana-metrics = { path = "../metrics", version = "=1.8.1" }
+solana-measure = { path = "../measure", version = "=1.8.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.1" }
+solana-perf = { path = "../perf", version = "=1.8.1" }
+solana-poh = { path = "../poh", version = "=1.8.1" }
+solana-program-test = { path = "../program-test", version = "=1.8.1" }
+solana-rpc = { path = "../rpc", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.8.1" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.8.1" }
+solana-streamer = { path = "../streamer", version = "=1.8.1" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.8.1" }
 spl-token-v2-0 = { package = "spl-token", version = "=3.2.0", features = ["no-entrypoint"] }
 tempfile = "3.1.0"
 thiserror = "1.0"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.7.15" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.8.1" }
 trees = "0.2.1"
 
 [dev-dependencies]
@@ -82,8 +84,8 @@ num_cpus = "1.13.0"
 reqwest = { version = "0.11.2", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 serde_json = "1.0.56"
 serial_test = "0.4.0"
-solana-stake-program = { path = "../programs/stake", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
+solana-stake-program = { path = "../programs/stake", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
 symlink = "0.1.0"
 systemstat = "0.1.5"
 tokio = { version = "1", features = ["full"] }

core/benches/banking_stage.rs

@@ -18,6 +18,8 @@ use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_poh::poh_recorder::{create_test_recorder, WorkingBankEntry};
 use solana_runtime::bank::Bank;
+use solana_runtime::cost_model::CostModel;
+use solana_runtime::cost_tracker_stats::CostTrackerStats;
 use solana_sdk::genesis_config::GenesisConfig;
 use solana_sdk::hash::Hash;
 use solana_sdk::message::Message;
@@ -33,7 +35,7 @@ use solana_streamer::socket::SocketAddrSpace;
 use std::collections::VecDeque;
 use std::sync::atomic::Ordering;
 use std::sync::mpsc::Receiver;
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
 use std::time::{Duration, Instant};
 use test::Bencher;
 
@@ -92,6 +94,8 @@ fn bench_consume_buffered(bencher: &mut Bencher) {
                 None::<Box<dyn Fn()>>,
                 &BankingStageStats::default(),
                 &recorder,
+                &Arc::new(RwLock::new(CostModel::default())),
+                &mut CostTrackerStats::default(),
             );
         });
 
@@ -165,6 +169,11 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) {
     bank.ns_per_slot = std::u128::MAX;
     let bank = Arc::new(Bank::new(&genesis_config));
 
+    // set cost tracker limits to MAX so it will not filter out TXs
+    bank.write_cost_tracker()
+        .unwrap()
+        .set_limits(std::u64::MAX, std::u64::MAX);
+
     debug!("threads: {} txs: {}", num_threads, txes);
 
     let transactions = match tx_type {
@@ -218,6 +227,7 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) {
             vote_receiver,
             None,
             s,
+            Arc::new(RwLock::new(CostModel::default())),
         );
         poh_recorder.lock().unwrap().set_bank(&bank);
 

core/benches/cluster_info.rs

@@ -75,9 +75,9 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
             &cluster_nodes_cache,
             &last_datapoint,
             &mut TransmitShredsStats::default(),
-            &SocketAddrSpace::Unspecified,
             &cluster_info,
             &bank_forks,
+            &SocketAddrSpace::Unspecified,
         )
         .unwrap();
     });

core/src/banking_stage.rs

@@ -24,6 +24,9 @@ use solana_runtime::{
         TransactionExecutionResult,
     },
     bank_utils,
+    cost_model::CostModel,
+    cost_tracker::CostTracker,
+    cost_tracker_stats::CostTrackerStats,
     hashed_transaction::HashedTransaction,
     transaction_batch::TransactionBatch,
     vote_sender_types::ReplayVoteSender,
@@ -33,6 +36,7 @@ use solana_sdk::{
         Slot, DEFAULT_TICKS_PER_SLOT, MAX_PROCESSING_AGE, MAX_TRANSACTION_FORWARDING_DELAY,
         MAX_TRANSACTION_FORWARDING_DELAY_GPU,
     },
+    feature_set,
     message::Message,
     pubkey::Pubkey,
     short_vec::decode_shortu16_len,
@@ -52,7 +56,7 @@ use std::{
     net::{SocketAddr, UdpSocket},
     ops::DerefMut,
     sync::atomic::{AtomicU64, AtomicUsize, Ordering},
-    sync::{Arc, Mutex},
+    sync::{Arc, Mutex, RwLock, RwLockReadGuard},
     thread::{self, Builder, JoinHandle},
     time::Duration,
     time::Instant,
@@ -93,6 +97,8 @@ pub struct BankingStageStats {
     current_buffered_packet_batches_count: AtomicUsize,
     rebuffered_packets_count: AtomicUsize,
     consumed_buffered_packets_count: AtomicUsize,
+    cost_tracker_check_count: AtomicUsize,
+    cost_forced_retry_transactions_count: AtomicUsize,
 
     // Timing
     consume_buffered_packets_elapsed: AtomicU64,
@@ -101,7 +107,11 @@ pub struct BankingStageStats {
     filter_pending_packets_elapsed: AtomicU64,
     packet_duplicate_check_elapsed: AtomicU64,
     packet_conversion_elapsed: AtomicU64,
+    unprocessed_packet_conversion_elapsed: AtomicU64,
     transaction_processing_elapsed: AtomicU64,
+    cost_tracker_update_elapsed: AtomicU64,
+    cost_tracker_clone_elapsed: AtomicU64,
+    cost_tracker_check_elapsed: AtomicU64,
 }
 
 impl BankingStageStats {
@@ -165,6 +175,17 @@ impl BankingStageStats {
                         .swap(0, Ordering::Relaxed) as i64,
                     i64
                 ),
+                (
+                    "cost_tracker_check_count",
+                    self.cost_tracker_check_count.swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+                (
+                    "cost_forced_retry_transactions_count",
+                    self.cost_forced_retry_transactions_count
+                        .swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
                 (
                     "consume_buffered_packets_elapsed",
                     self.consume_buffered_packets_elapsed
@@ -199,12 +220,33 @@ impl BankingStageStats {
                     self.packet_conversion_elapsed.swap(0, Ordering::Relaxed) as i64,
                     i64
                 ),
+                (
+                    "unprocessed_packet_conversion_elapsed",
+                    self.unprocessed_packet_conversion_elapsed
+                        .swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
                 (
                     "transaction_processing_elapsed",
                     self.transaction_processing_elapsed
                         .swap(0, Ordering::Relaxed) as i64,
                     i64
                 ),
+                (
+                    "cost_tracker_update_elapsed",
+                    self.cost_tracker_update_elapsed.swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+                (
+                    "cost_tracker_clone_elapsed",
+                    self.cost_tracker_clone_elapsed.swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+                (
+                    "cost_tracker_check_elapsed",
+                    self.cost_tracker_check_elapsed.swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
             );
         }
     }
@@ -241,6 +283,7 @@ impl BankingStage {
         verified_vote_receiver: CrossbeamReceiver<Vec<Packets>>,
         transaction_status_sender: Option<TransactionStatusSender>,
         gossip_vote_sender: ReplayVoteSender,
+        cost_model: Arc<RwLock<CostModel>>,
     ) -> Self {
         Self::new_num_threads(
             cluster_info,
@@ -251,6 +294,7 @@ impl BankingStage {
             Self::num_threads(),
             transaction_status_sender,
             gossip_vote_sender,
+            cost_model,
         )
     }
 
@@ -263,6 +307,7 @@ impl BankingStage {
         num_threads: u32,
         transaction_status_sender: Option<TransactionStatusSender>,
         gossip_vote_sender: ReplayVoteSender,
+        cost_model: Arc<RwLock<CostModel>>,
     ) -> Self {
         let batch_limit = TOTAL_BUFFERED_PACKETS / ((num_threads - 1) as usize * PACKETS_PER_BATCH);
         // Single thread to generate entries from many banks.
@@ -298,6 +343,7 @@ impl BankingStage {
                 let gossip_vote_sender = gossip_vote_sender.clone();
                 let duplicates = duplicates.clone();
                 let data_budget = data_budget.clone();
+                let cost_model = cost_model.clone();
                 Builder::new()
                     .name("solana-banking-stage-tx".to_string())
                     .spawn(move || {
@@ -314,6 +360,7 @@ impl BankingStage {
                             gossip_vote_sender,
                             &duplicates,
                             &data_budget,
+                            cost_model,
                         );
                     })
                     .unwrap()
@@ -371,6 +418,7 @@ impl BankingStage {
         has_more_unprocessed_transactions
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub fn consume_buffered_packets(
         my_pubkey: &Pubkey,
         max_tx_ingestion_ns: u128,
@@ -381,6 +429,8 @@ impl BankingStage {
         test_fn: Option<impl Fn()>,
         banking_stage_stats: &BankingStageStats,
         recorder: &TransactionRecorder,
+        cost_model: &Arc<RwLock<CostModel>>,
+        cost_tracker_stats: &mut CostTrackerStats,
     ) {
         let mut rebuffered_packets_len = 0;
         let mut new_tx_count = 0;
@@ -398,6 +448,9 @@ impl BankingStage {
                     original_unprocessed_indexes,
                     my_pubkey,
                     *next_leader,
+                    banking_stage_stats,
+                    cost_model,
+                    cost_tracker_stats,
                 );
                 Self::update_buffered_packets_with_new_unprocessed(
                     original_unprocessed_indexes,
@@ -416,6 +469,8 @@ impl BankingStage {
                             transaction_status_sender.clone(),
                             gossip_vote_sender,
                             banking_stage_stats,
+                            cost_model,
+                            cost_tracker_stats,
                         );
                     if processed < verified_txs_len
                         || !Bank::should_bank_still_be_processing_txs(
@@ -519,6 +574,8 @@ impl BankingStage {
         banking_stage_stats: &BankingStageStats,
         recorder: &TransactionRecorder,
         data_budget: &DataBudget,
+        cost_model: &Arc<RwLock<CostModel>>,
+        cost_tracker_stats: &mut CostTrackerStats,
     ) -> BufferedPacketsDecision {
         let bank_start;
         let (
@@ -559,6 +616,8 @@ impl BankingStage {
                     None::<Box<dyn Fn()>>,
                     banking_stage_stats,
                     recorder,
+                    cost_model,
+                    cost_tracker_stats,
                 );
             }
             BufferedPacketsDecision::Forward => {
@@ -638,11 +697,13 @@ impl BankingStage {
         gossip_vote_sender: ReplayVoteSender,
         duplicates: &Arc<Mutex<(LruCache<u64, ()>, PacketHasher)>>,
         data_budget: &DataBudget,
+        cost_model: Arc<RwLock<CostModel>>,
     ) {
         let recorder = poh_recorder.lock().unwrap().recorder();
         let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
         let mut buffered_packets = VecDeque::with_capacity(batch_limit);
         let banking_stage_stats = BankingStageStats::new(id);
+        let mut cost_tracker_stats = CostTrackerStats::new(id, 0);
         loop {
             while !buffered_packets.is_empty() {
                 let decision = Self::process_buffered_packets(
@@ -657,6 +718,8 @@ impl BankingStage {
                     &banking_stage_stats,
                     &recorder,
                     data_budget,
+                    &cost_model,
+                    &mut cost_tracker_stats,
                 );
                 if matches!(decision, BufferedPacketsDecision::Hold)
                     || matches!(decision, BufferedPacketsDecision::ForwardAndHold)
@@ -691,6 +754,8 @@ impl BankingStage {
                 &banking_stage_stats,
                 duplicates,
                 &recorder,
+                &cost_model,
+                &mut cost_tracker_stats,
             ) {
                 Ok(()) | Err(RecvTimeoutError::Timeout) => (),
                 Err(RecvTimeoutError::Disconnected) => break,
@@ -794,7 +859,6 @@ impl BankingStage {
         };
 
         let mut execute_timings = ExecuteTimings::default();
-
         let (
             mut loaded_accounts,
             results,
@@ -935,12 +999,12 @@ impl BankingStage {
     ) -> (usize, Vec<usize>) {
         let mut chunk_start = 0;
         let mut unprocessed_txs = vec![];
+
         while chunk_start != transactions.len() {
             let chunk_end = std::cmp::min(
                 transactions.len(),
                 chunk_start + MAX_NUM_TRANSACTIONS_PER_BATCH,
             );
-
             let (result, retryable_txs_in_chunk) = Self::process_and_record_transactions(
                 bank,
                 &transactions[chunk_start..chunk_end],
@@ -1023,13 +1087,22 @@ impl BankingStage {
     // This function deserializes packets into transactions, computes the blake3 hash of transaction messages,
     // and verifies secp256k1 instructions. A list of valid transactions are returned with their message hashes
     // and packet indexes.
+    // Also returned is packet indexes for transaction should be retried due to cost limits.
+    #[allow(clippy::needless_collect)]
     fn transactions_from_packets(
         msgs: &Packets,
         transaction_indexes: &[usize],
-        libsecp256k1_0_5_upgrade_enabled: bool,
+        feature_set: &Arc<feature_set::FeatureSet>,
+        read_cost_tracker: &RwLockReadGuard<CostTracker>,
+        banking_stage_stats: &BankingStageStats,
+        demote_program_write_locks: bool,
         votes_only: bool,
-    ) -> (Vec<HashedTransaction<'static>>, Vec<usize>) {
-        transaction_indexes
+        cost_model: &Arc<RwLock<CostModel>>,
+        cost_tracker_stats: &mut CostTrackerStats,
+    ) -> (Vec<HashedTransaction<'static>>, Vec<usize>, Vec<usize>) {
+        let mut retryable_transaction_packet_indexes: Vec<usize> = vec![];
+
+        let verified_transactions_with_packet_indexes: Vec<_> = transaction_indexes
             .iter()
             .filter_map(|tx_index| {
                 let p = &msgs.packets[*tx_index];
@@ -1038,16 +1111,71 @@ impl BankingStage {
                 }
 
                 let tx: Transaction = limited_deserialize(&p.data[0..p.meta.size]).ok()?;
-                tx.verify_precompiles(libsecp256k1_0_5_upgrade_enabled)
-                    .ok()?;
-                let message_bytes = Self::packet_message(p)?;
-                let message_hash = Message::hash_raw_message(message_bytes);
-                Some((
-                    HashedTransaction::new(Cow::Owned(tx), message_hash),
-                    tx_index,
-                ))
+                tx.verify_precompiles(feature_set).ok()?;
+
+                Some((tx, *tx_index))
             })
-            .unzip()
+            .collect();
+        banking_stage_stats.cost_tracker_check_count.fetch_add(
+            verified_transactions_with_packet_indexes.len(),
+            Ordering::Relaxed,
+        );
+
+        let mut cost_tracker_check_time = Measure::start("cost_tracker_check_time");
+        let filtered_transactions_with_packet_indexes: Vec<_> = {
+            verified_transactions_with_packet_indexes
+                .into_iter()
+                .filter_map(|(tx, tx_index)| {
+                    // put transaction into retry queue if it wouldn't fit
+                    // into current bank
+                    let is_vote = &msgs.packets[tx_index].meta.is_simple_vote_tx;
+
+                    // excluding vote TX from cost_model, for now
+                    if !is_vote
+                        && read_cost_tracker
+                            .would_transaction_fit(
+                                &tx,
+                                &cost_model
+                                    .read()
+                                    .unwrap()
+                                    .calculate_cost(&tx, demote_program_write_locks),
+                                cost_tracker_stats,
+                            )
+                            .is_err()
+                    {
+                        debug!("transaction {:?} would exceed limit", tx);
+                        retryable_transaction_packet_indexes.push(tx_index);
+                        return None;
+                    }
+                    Some((tx, tx_index))
+                })
+                .collect()
+        };
+        cost_tracker_check_time.stop();
+
+        let (filtered_transactions, filter_transaction_packet_indexes) =
+            filtered_transactions_with_packet_indexes
+                .into_iter()
+                .filter_map(|(tx, tx_index)| {
+                    let p = &msgs.packets[tx_index];
+                    let message_bytes = Self::packet_message(p)?;
+                    let message_hash = Message::hash_raw_message(message_bytes);
+                    Some((
+                        HashedTransaction::new(Cow::Owned(tx), message_hash),
+                        tx_index,
+                    ))
+                })
+                .unzip();
+
+        banking_stage_stats
+            .cost_tracker_check_elapsed
+            .fetch_add(cost_tracker_check_time.as_us(), Ordering::Relaxed);
+
+        (
+            filtered_transactions,
+            filter_transaction_packet_indexes,
+            retryable_transaction_packet_indexes,
+        )
     }
 
     /// This function filters pending packets that are still valid
@@ -1089,6 +1217,7 @@ impl BankingStage {
         Self::filter_valid_transaction_indexes(&results, transaction_to_packet_indexes)
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn process_packets_transactions(
         bank: &Arc<Bank>,
         bank_creation_time: &Instant,
@@ -1098,20 +1227,33 @@ impl BankingStage {
         transaction_status_sender: Option<TransactionStatusSender>,
         gossip_vote_sender: &ReplayVoteSender,
         banking_stage_stats: &BankingStageStats,
+        cost_model: &Arc<RwLock<CostModel>>,
+        cost_tracker_stats: &mut CostTrackerStats,
     ) -> (usize, usize, Vec<usize>) {
         let mut packet_conversion_time = Measure::start("packet_conversion");
-        let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets(
-            msgs,
-            &packet_indexes,
-            bank.libsecp256k1_0_5_upgrade_enabled(),
-            bank.vote_only_bank(),
-        );
+        let (transactions, transaction_to_packet_indexes, retryable_packet_indexes) =
+            Self::transactions_from_packets(
+                msgs,
+                &packet_indexes,
+                &bank.feature_set,
+                &bank.read_cost_tracker().unwrap(),
+                banking_stage_stats,
+                bank.demote_program_write_locks(),
+                bank.vote_only_bank(),
+                cost_model,
+                cost_tracker_stats,
+            );
         packet_conversion_time.stop();
+        inc_new_counter_info!("banking_stage-packet_conversion", 1);
 
+        banking_stage_stats
+            .cost_forced_retry_transactions_count
+            .fetch_add(retryable_packet_indexes.len(), Ordering::Relaxed);
         debug!(
-            "bank: {} filtered transactions {}",
+            "bank: {} filtered transactions {} cost limited transactions {}",
             bank.slot(),
-            transactions.len()
+            transactions.len(),
+            retryable_packet_indexes.len()
         );
 
         let tx_len = transactions.len();
@@ -1126,11 +1268,30 @@ impl BankingStage {
             gossip_vote_sender,
         );
         process_tx_time.stop();
-
         let unprocessed_tx_count = unprocessed_tx_indexes.len();
+        inc_new_counter_info!(
+            "banking_stage-unprocessed_transactions",
+            unprocessed_tx_count
+        );
+
+        // applying cost of processed transactions to shared cost_tracker
+        let mut cost_tracking_time = Measure::start("cost_tracking_time");
+        transactions.iter().enumerate().for_each(|(index, tx)| {
+            if unprocessed_tx_indexes.iter().all(|&i| i != index) {
+                bank.write_cost_tracker().unwrap().add_transaction_cost(
+                    tx.transaction(),
+                    &cost_model
+                        .read()
+                        .unwrap()
+                        .calculate_cost(tx.transaction(), bank.demote_program_write_locks()),
+                    cost_tracker_stats,
+                );
+            }
+        });
+        cost_tracking_time.stop();
 
         let mut filter_pending_packets_time = Measure::start("filter_pending_packets_time");
-        let filtered_unprocessed_packet_indexes = Self::filter_pending_packets_from_pending_txs(
+        let mut filtered_unprocessed_packet_indexes = Self::filter_pending_packets_from_pending_txs(
             bank,
             &transactions,
             &transaction_to_packet_indexes,
@@ -1143,12 +1304,19 @@ impl BankingStage {
             unprocessed_tx_count.saturating_sub(filtered_unprocessed_packet_indexes.len())
         );
 
+        // combine cost-related unprocessed transactions with bank determined unprocessed for
+        // buffering
+        filtered_unprocessed_packet_indexes.extend(retryable_packet_indexes);
+
         banking_stage_stats
             .packet_conversion_elapsed
             .fetch_add(packet_conversion_time.as_us(), Ordering::Relaxed);
         banking_stage_stats
             .transaction_processing_elapsed
             .fetch_add(process_tx_time.as_us(), Ordering::Relaxed);
+        banking_stage_stats
+            .cost_tracker_update_elapsed
+            .fetch_add(cost_tracking_time.as_us(), Ordering::Relaxed);
         banking_stage_stats
             .filter_pending_packets_elapsed
             .fetch_add(filter_pending_packets_time.as_us(), Ordering::Relaxed);
@@ -1162,6 +1330,9 @@ impl BankingStage {
         transaction_indexes: &[usize],
         my_pubkey: &Pubkey,
         next_leader: Option<Pubkey>,
+        banking_stage_stats: &BankingStageStats,
+        cost_model: &Arc<RwLock<CostModel>>,
+        cost_tracker_stats: &mut CostTrackerStats,
     ) -> Vec<usize> {
         // Check if we are the next leader. If so, let's not filter the packets
         // as we'll filter it again while processing the packets.
@@ -1172,27 +1343,44 @@ impl BankingStage {
             }
         }
 
-        let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets(
-            msgs,
-            &transaction_indexes,
-            bank.libsecp256k1_0_5_upgrade_enabled(),
-            bank.vote_only_bank(),
-        );
+        let mut unprocessed_packet_conversion_time =
+            Measure::start("unprocessed_packet_conversion");
+        let (transactions, transaction_to_packet_indexes, retry_packet_indexes) =
+            Self::transactions_from_packets(
+                msgs,
+                &transaction_indexes,
+                &bank.feature_set,
+                &bank.read_cost_tracker().unwrap(),
+                banking_stage_stats,
+                bank.demote_program_write_locks(),
+                bank.vote_only_bank(),
+                cost_model,
+                cost_tracker_stats,
+            );
+        unprocessed_packet_conversion_time.stop();
 
         let tx_count = transaction_to_packet_indexes.len();
 
         let unprocessed_tx_indexes = (0..transactions.len()).collect_vec();
-        let filtered_unprocessed_packet_indexes = Self::filter_pending_packets_from_pending_txs(
+        let mut filtered_unprocessed_packet_indexes = Self::filter_pending_packets_from_pending_txs(
             bank,
             &transactions,
             &transaction_to_packet_indexes,
             &unprocessed_tx_indexes,
         );
 
+        filtered_unprocessed_packet_indexes.extend(retry_packet_indexes);
+
         inc_new_counter_info!(
             "banking_stage-dropped_tx_before_forwarding",
             tx_count.saturating_sub(filtered_unprocessed_packet_indexes.len())
         );
+        banking_stage_stats
+            .unprocessed_packet_conversion_elapsed
+            .fetch_add(
+                unprocessed_packet_conversion_time.as_us(),
+                Ordering::Relaxed,
+            );
 
         filtered_unprocessed_packet_indexes
     }
@@ -1228,6 +1416,8 @@ impl BankingStage {
         banking_stage_stats: &BankingStageStats,
         duplicates: &Arc<Mutex<(LruCache<u64, ()>, PacketHasher)>>,
         recorder: &TransactionRecorder,
+        cost_model: &Arc<RwLock<CostModel>>,
+        cost_tracker_stats: &mut CostTrackerStats,
     ) -> Result<(), RecvTimeoutError> {
         let mut recv_time = Measure::start("process_packets_recv");
         let mms = verified_receiver.recv_timeout(recv_timeout)?;
@@ -1279,6 +1469,8 @@ impl BankingStage {
                     transaction_status_sender.clone(),
                     gossip_vote_sender,
                     banking_stage_stats,
+                    cost_model,
+                    cost_tracker_stats,
                 );
 
             new_tx_count += processed;
@@ -1310,6 +1502,9 @@ impl BankingStage {
                         &packet_indexes,
                         my_pubkey,
                         next_leader,
+                        banking_stage_stats,
+                        cost_model,
+                        cost_tracker_stats,
                     );
                     Self::push_unprocessed(
                         buffered_packets,
@@ -1480,6 +1675,7 @@ mod tests {
         poh_service::PohService,
     };
     use solana_rpc::transaction_status_service::TransactionStatusService;
+    use solana_runtime::cost_model::CostModel;
     use solana_sdk::{
         hash::Hash,
         instruction::InstructionError,
@@ -1536,6 +1732,7 @@ mod tests {
                 gossip_verified_vote_receiver,
                 None,
                 vote_forward_sender,
+                Arc::new(RwLock::new(CostModel::default())),
             );
             drop(verified_sender);
             drop(gossip_verified_vote_sender);
@@ -1584,6 +1781,7 @@ mod tests {
                 verified_gossip_vote_receiver,
                 None,
                 vote_forward_sender,
+                Arc::new(RwLock::new(CostModel::default())),
             );
             trace!("sending bank");
             drop(verified_sender);
@@ -1656,6 +1854,7 @@ mod tests {
                 gossip_verified_vote_receiver,
                 None,
                 gossip_vote_sender,
+                Arc::new(RwLock::new(CostModel::default())),
             );
 
             // fund another account so we can send 2 good transactions in a single batch.
@@ -1806,6 +2005,7 @@ mod tests {
                     3,
                     None,
                     gossip_vote_sender,
+                    Arc::new(RwLock::new(CostModel::default())),
                 );
 
                 // wait for banking_stage to eat the packets
@@ -2627,6 +2827,8 @@ mod tests {
                 None::<Box<dyn Fn()>>,
                 &BankingStageStats::default(),
                 &recorder,
+                &Arc::new(RwLock::new(CostModel::default())),
+                &mut CostTrackerStats::default(),
             );
             assert_eq!(buffered_packets[0].1.len(), num_conflicting_transactions);
             // When the poh recorder has a bank, should process all non conflicting buffered packets.
@@ -2643,6 +2845,8 @@ mod tests {
                     None::<Box<dyn Fn()>>,
                     &BankingStageStats::default(),
                     &recorder,
+                    &Arc::new(RwLock::new(CostModel::default())),
+                    &mut CostTrackerStats::default(),
                 );
                 if num_expected_unprocessed == 0 {
                     assert!(buffered_packets.is_empty())
@@ -2708,6 +2912,8 @@ mod tests {
                         test_fn,
                         &BankingStageStats::default(),
                         &recorder,
+                        &Arc::new(RwLock::new(CostModel::default())),
+                        &mut CostTrackerStats::default(),
                     );
 
                     // Check everything is correct. All indexes after `interrupted_iteration`
@@ -2956,22 +3162,34 @@ mod tests {
                 make_test_packets(vec![transfer_tx.clone(), transfer_tx.clone()], vote_indexes);
 
             let mut votes_only = false;
-            let (txs, tx_packet_index) = BankingStage::transactions_from_packets(
-                &packets,
-                &packet_indexes,
-                false,
-                votes_only,
-            );
+            let (txs, tx_packet_index, _retryable_packet_indexes) =
+                BankingStage::transactions_from_packets(
+                    &packets,
+                    &packet_indexes,
+                    &Arc::new(feature_set::FeatureSet::default()),
+                    &RwLock::new(CostTracker::default()).read().unwrap(),
+                    &BankingStageStats::default(),
+                    false,
+                    votes_only,
+                    &Arc::new(RwLock::new(CostModel::default())),
+                    &mut CostTrackerStats::default(),
+                );
             assert_eq!(2, txs.len());
             assert_eq!(vec![0, 1], tx_packet_index);
 
             votes_only = true;
-            let (txs, tx_packet_index) = BankingStage::transactions_from_packets(
-                &packets,
-                &packet_indexes,
-                false,
-                votes_only,
-            );
+            let (txs, tx_packet_index, _retryable_packet_indexes) =
+                BankingStage::transactions_from_packets(
+                    &packets,
+                    &packet_indexes,
+                    &Arc::new(feature_set::FeatureSet::default()),
+                    &RwLock::new(CostTracker::default()).read().unwrap(),
+                    &BankingStageStats::default(),
+                    false,
+                    votes_only,
+                    &Arc::new(RwLock::new(CostModel::default())),
+                    &mut CostTrackerStats::default(),
+                );
             assert_eq!(0, txs.len());
             assert_eq!(0, tx_packet_index.len());
         }
@@ -2985,22 +3203,34 @@ mod tests {
             );
 
             let mut votes_only = false;
-            let (txs, tx_packet_index) = BankingStage::transactions_from_packets(
-                &packets,
-                &packet_indexes,
-                false,
-                votes_only,
-            );
+            let (txs, tx_packet_index, _retryable_packet_indexes) =
+                BankingStage::transactions_from_packets(
+                    &packets,
+                    &packet_indexes,
+                    &Arc::new(feature_set::FeatureSet::default()),
+                    &RwLock::new(CostTracker::default()).read().unwrap(),
+                    &BankingStageStats::default(),
+                    false,
+                    votes_only,
+                    &Arc::new(RwLock::new(CostModel::default())),
+                    &mut CostTrackerStats::default(),
+                );
             assert_eq!(3, txs.len());
             assert_eq!(vec![0, 1, 2], tx_packet_index);
 
             votes_only = true;
-            let (txs, tx_packet_index) = BankingStage::transactions_from_packets(
-                &packets,
-                &packet_indexes,
-                false,
-                votes_only,
-            );
+            let (txs, tx_packet_index, _retryable_packet_indexes) =
+                BankingStage::transactions_from_packets(
+                    &packets,
+                    &packet_indexes,
+                    &Arc::new(feature_set::FeatureSet::default()),
+                    &RwLock::new(CostTracker::default()).read().unwrap(),
+                    &BankingStageStats::default(),
+                    false,
+                    votes_only,
+                    &Arc::new(RwLock::new(CostModel::default())),
+                    &mut CostTrackerStats::default(),
+                );
             assert_eq!(2, txs.len());
             assert_eq!(vec![0, 2], tx_packet_index);
         }
@@ -3014,22 +3244,34 @@ mod tests {
             );
 
             let mut votes_only = false;
-            let (txs, tx_packet_index) = BankingStage::transactions_from_packets(
-                &packets,
-                &packet_indexes,
-                false,
-                votes_only,
-            );
+            let (txs, tx_packet_index, _retryable_packet_indexes) =
+                BankingStage::transactions_from_packets(
+                    &packets,
+                    &packet_indexes,
+                    &Arc::new(feature_set::FeatureSet::default()),
+                    &RwLock::new(CostTracker::default()).read().unwrap(),
+                    &BankingStageStats::default(),
+                    false,
+                    votes_only,
+                    &Arc::new(RwLock::new(CostModel::default())),
+                    &mut CostTrackerStats::default(),
+                );
             assert_eq!(3, txs.len());
             assert_eq!(vec![0, 1, 2], tx_packet_index);
 
             votes_only = true;
-            let (txs, tx_packet_index) = BankingStage::transactions_from_packets(
-                &packets,
-                &packet_indexes,
-                false,
-                votes_only,
-            );
+            let (txs, tx_packet_index, _retryable_packet_indexes) =
+                BankingStage::transactions_from_packets(
+                    &packets,
+                    &packet_indexes,
+                    &Arc::new(feature_set::FeatureSet::default()),
+                    &RwLock::new(CostTracker::default()).read().unwrap(),
+                    &BankingStageStats::default(),
+                    false,
+                    votes_only,
+                    &Arc::new(RwLock::new(CostModel::default())),
+                    &mut CostTrackerStats::default(),
+                );
             assert_eq!(3, txs.len());
             assert_eq!(vec![0, 1, 2], tx_packet_index);
         }

core/src/broadcast_stage.rs

@@ -22,8 +22,10 @@ use {
     solana_metrics::{inc_new_counter_error, inc_new_counter_info},
     solana_poh::poh_recorder::WorkingBankEntry,
     solana_runtime::{bank::Bank, bank_forks::BankForks},
-    solana_sdk::timing::{timestamp, AtomicInterval},
-    solana_sdk::{clock::Slot, pubkey::Pubkey},
+    solana_sdk::{
+        timing::{timestamp, AtomicInterval},
+        {clock::Slot, pubkey::Pubkey},
+    },
     solana_streamer::{
         sendmmsg::{batch_send, SendPktsError},
         socket::SocketAddrSpace,
@@ -31,9 +33,11 @@ use {
     std::{
         collections::HashMap,
         net::UdpSocket,
-        sync::atomic::{AtomicBool, Ordering},
-        sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender},
-        sync::{Arc, Mutex, RwLock},
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender},
+            Arc, Mutex, RwLock,
+        },
         thread::{self, Builder, JoinHandle},
         time::{Duration, Instant},
     },
@@ -399,9 +403,9 @@ pub fn broadcast_shreds(
     cluster_nodes_cache: &ClusterNodesCache<BroadcastStage>,
     last_datapoint_submit: &Arc<AtomicInterval>,
     transmit_stats: &mut TransmitShredsStats,
-    socket_addr_space: &SocketAddrSpace,
     cluster_info: &ClusterInfo,
     bank_forks: &Arc<RwLock<BankForks>>,
+    socket_addr_space: &SocketAddrSpace,
 ) -> Result<()> {
     let mut result = Ok(());
     let mut shred_select = Measure::start("shred_select");

core/src/broadcast_stage/broadcast_fake_shreds_run.rs

@@ -89,6 +89,7 @@ impl BroadcastRun for BroadcastFakeShredsRun {
             slot,
             num_expected_batches: None,
             slot_start_ts: Instant::now(),
+            was_interrupted: false,
         };
         // 3) Start broadcast step
         //some indicates fake shreds

core/src/broadcast_stage/broadcast_metrics.rs

@@ -2,7 +2,7 @@ use super::*;
 
 pub(crate) trait BroadcastStats {
     fn update(&mut self, new_stats: &Self);
-    fn report_stats(&mut self, slot: Slot, slot_start: Instant);
+    fn report_stats(&mut self, slot: Slot, slot_start: Instant, was_interrupted: bool);
 }
 
 #[derive(Clone)]
@@ -10,6 +10,7 @@ pub(crate) struct BroadcastShredBatchInfo {
     pub(crate) slot: Slot,
     pub(crate) num_expected_batches: Option<usize>,
     pub(crate) slot_start_ts: Instant,
+    pub(crate) was_interrupted: bool,
 }
 
 #[derive(Default, Clone)]
@@ -33,25 +34,39 @@ impl BroadcastStats for TransmitShredsStats {
         self.total_packets += new_stats.total_packets;
         self.dropped_packets += new_stats.dropped_packets;
     }
-    fn report_stats(&mut self, slot: Slot, slot_start: Instant) {
-        datapoint_info!(
-            "broadcast-transmit-shreds-stats",
-            ("slot", slot as i64, i64),
-            (
-                "end_to_end_elapsed",
-                // `slot_start` signals when the first batch of shreds was
-                // received, used to measure duration of broadcast
-                slot_start.elapsed().as_micros() as i64,
-                i64
-            ),
-            ("transmit_elapsed", self.transmit_elapsed as i64, i64),
-            ("send_mmsg_elapsed", self.send_mmsg_elapsed as i64, i64),
-            ("get_peers_elapsed", self.get_peers_elapsed as i64, i64),
-            ("num_shreds", self.num_shreds as i64, i64),
-            ("shred_select", self.shred_select as i64, i64),
-            ("total_packets", self.total_packets as i64, i64),
-            ("dropped_packets", self.dropped_packets as i64, i64),
-        );
+    fn report_stats(&mut self, slot: Slot, slot_start: Instant, was_interrupted: bool) {
+        if was_interrupted {
+            datapoint_info!(
+                "broadcast-transmit-shreds-interrupted-stats",
+                ("slot", slot as i64, i64),
+                ("transmit_elapsed", self.transmit_elapsed as i64, i64),
+                ("send_mmsg_elapsed", self.send_mmsg_elapsed as i64, i64),
+                ("get_peers_elapsed", self.get_peers_elapsed as i64, i64),
+                ("num_shreds", self.num_shreds as i64, i64),
+                ("shred_select", self.shred_select as i64, i64),
+                ("total_packets", self.total_packets as i64, i64),
+                ("dropped_packets", self.dropped_packets as i64, i64),
+            );
+        } else {
+            datapoint_info!(
+                "broadcast-transmit-shreds-stats",
+                ("slot", slot as i64, i64),
+                (
+                    "end_to_end_elapsed",
+                    // `slot_start` signals when the first batch of shreds was
+                    // received, used to measure duration of broadcast
+                    slot_start.elapsed().as_micros() as i64,
+                    i64
+                ),
+                ("transmit_elapsed", self.transmit_elapsed as i64, i64),
+                ("send_mmsg_elapsed", self.send_mmsg_elapsed as i64, i64),
+                ("get_peers_elapsed", self.get_peers_elapsed as i64, i64),
+                ("num_shreds", self.num_shreds as i64, i64),
+                ("shred_select", self.shred_select as i64, i64),
+                ("total_packets", self.total_packets as i64, i64),
+                ("dropped_packets", self.dropped_packets as i64, i64),
+            );
+        }
     }
 }
 
@@ -65,24 +80,37 @@ impl BroadcastStats for InsertShredsStats {
         self.insert_shreds_elapsed += new_stats.insert_shreds_elapsed;
         self.num_shreds += new_stats.num_shreds;
     }
-    fn report_stats(&mut self, slot: Slot, slot_start: Instant) {
-        datapoint_info!(
-            "broadcast-insert-shreds-stats",
-            ("slot", slot as i64, i64),
-            (
-                "end_to_end_elapsed",
-                // `slot_start` signals when the first batch of shreds was
-                // received, used to measure duration of broadcast
-                slot_start.elapsed().as_micros() as i64,
-                i64
-            ),
-            (
-                "insert_shreds_elapsed",
-                self.insert_shreds_elapsed as i64,
-                i64
-            ),
-            ("num_shreds", self.num_shreds as i64, i64),
-        );
+    fn report_stats(&mut self, slot: Slot, slot_start: Instant, was_interrupted: bool) {
+        if was_interrupted {
+            datapoint_info!(
+                "broadcast-insert-shreds-interrupted-stats",
+                ("slot", slot as i64, i64),
+                (
+                    "insert_shreds_elapsed",
+                    self.insert_shreds_elapsed as i64,
+                    i64
+                ),
+                ("num_shreds", self.num_shreds as i64, i64),
+            );
+        } else {
+            datapoint_info!(
+                "broadcast-insert-shreds-stats",
+                ("slot", slot as i64, i64),
+                (
+                    "end_to_end_elapsed",
+                    // `slot_start` signals when the first batch of shreds was
+                    // received, used to measure duration of broadcast
+                    slot_start.elapsed().as_micros() as i64,
+                    i64
+                ),
+                (
+                    "insert_shreds_elapsed",
+                    self.insert_shreds_elapsed as i64,
+                    i64
+                ),
+                ("num_shreds", self.num_shreds as i64, i64),
+            );
+        }
     }
 }
 
@@ -128,9 +156,11 @@ impl<T: BroadcastStats + Default> SlotBroadcastStats<T> {
                 }
                 if let Some(num_expected_batches) = slot_batch_counter.num_expected_batches {
                     if slot_batch_counter.num_batches == num_expected_batches {
-                        slot_batch_counter
-                            .broadcast_shred_stats
-                            .report_stats(batch_info.slot, batch_info.slot_start_ts);
+                        slot_batch_counter.broadcast_shred_stats.report_stats(
+                            batch_info.slot,
+                            batch_info.slot_start_ts,
+                            batch_info.was_interrupted,
+                        );
                         should_delete = true;
                     }
                 }
@@ -159,7 +189,7 @@ mod test {
             self.count += new_stats.count;
             self.sender = new_stats.sender.clone();
         }
-        fn report_stats(&mut self, slot: Slot, slot_start: Instant) {
+        fn report_stats(&mut self, slot: Slot, slot_start: Instant, _was_interrupted: bool) {
             self.sender
                 .as_ref()
                 .unwrap()
@@ -186,6 +216,7 @@ mod test {
                 slot: 0,
                 num_expected_batches: Some(2),
                 slot_start_ts: start,
+                was_interrupted: false,
             }),
         );
 
@@ -242,6 +273,7 @@ mod test {
                 slot: 0,
                 num_expected_batches: None,
                 slot_start_ts: start,
+                was_interrupted: false,
             }),
         );
 
@@ -265,6 +297,7 @@ mod test {
                         slot,
                         num_expected_batches: None,
                         slot_start_ts: start,
+                        was_interrupted: false,
                     };
                     if i == round % num_threads {
                         broadcast_batch_info.num_expected_batches = Some(num_threads);

core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs

@@ -145,9 +145,9 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
             &self.cluster_nodes_cache,
             &Arc::new(AtomicInterval::default()),
             &mut TransmitShredsStats::default(),
-            cluster_info.socket_addr_space(),
             cluster_info,
             bank_forks,
+            cluster_info.socket_addr_space(),
         )
     }
     fn record(

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -97,7 +97,7 @@ impl StandardBroadcastRun {
                     stats,
                 );
                 shreds.insert(0, shred);
-                self.report_and_reset_stats();
+                self.report_and_reset_stats(true);
                 self.unfinished_slot = None;
                 shreds
             }
@@ -245,6 +245,7 @@ impl StandardBroadcastRun {
                     "Old broadcast start time for previous slot must exist if the previous slot
                  was interrupted",
                 ),
+                was_interrupted: true,
             });
             let shreds = Arc::new(prev_slot_shreds);
             debug_assert!(shreds.iter().all(|shred| shred.slot() == slot));
@@ -267,6 +268,7 @@ impl StandardBroadcastRun {
             slot_start_ts: self
                 .slot_broadcast_start
                 .expect("Start timestamp must exist for a slot if we're broadcasting the slot"),
+            was_interrupted: false,
         });
         get_leader_schedule_time.stop();
 
@@ -302,7 +304,7 @@ impl StandardBroadcastRun {
         self.process_shreds_stats.update(&process_stats);
 
         if last_tick_height == bank.max_tick_height() {
-            self.report_and_reset_stats();
+            self.report_and_reset_stats(false);
             self.unfinished_slot = None;
         }
 
@@ -362,9 +364,9 @@ impl StandardBroadcastRun {
             &self.cluster_nodes_cache,
             &self.last_datapoint_submit,
             &mut transmit_stats,
-            cluster_info.socket_addr_space(),
             cluster_info,
             bank_forks,
+            cluster_info.socket_addr_space(),
         )?;
         transmit_time.stop();
 
@@ -385,35 +387,59 @@ impl StandardBroadcastRun {
         transmit_shreds_stats.update(new_transmit_shreds_stats, broadcast_shred_batch_info);
     }
 
-    fn report_and_reset_stats(&mut self) {
+    fn report_and_reset_stats(&mut self, was_interrupted: bool) {
         let stats = &self.process_shreds_stats;
         let unfinished_slot = self.unfinished_slot.as_ref().unwrap();
-        datapoint_info!(
-            "broadcast-process-shreds-stats",
-            ("slot", unfinished_slot.slot as i64, i64),
-            ("shredding_time", stats.shredding_elapsed, i64),
-            ("receive_time", stats.receive_elapsed, i64),
-            (
-                "num_data_shreds",
-                unfinished_slot.next_shred_index as i64,
-                i64
-            ),
-            (
-                "slot_broadcast_time",
-                self.slot_broadcast_start.unwrap().elapsed().as_micros() as i64,
-                i64
-            ),
-            (
-                "get_leader_schedule_time",
-                stats.get_leader_schedule_elapsed,
-                i64
-            ),
-            ("serialize_shreds_time", stats.serialize_elapsed, i64),
-            ("gen_data_time", stats.gen_data_elapsed, i64),
-            ("gen_coding_time", stats.gen_coding_elapsed, i64),
-            ("sign_coding_time", stats.sign_coding_elapsed, i64),
-            ("coding_send_time", stats.coding_send_elapsed, i64),
-        );
+        if was_interrupted {
+            datapoint_info!(
+                "broadcast-process-shreds-interrupted-stats",
+                ("slot", unfinished_slot.slot as i64, i64),
+                ("shredding_time", stats.shredding_elapsed, i64),
+                ("receive_time", stats.receive_elapsed, i64),
+                (
+                    "num_data_shreds",
+                    unfinished_slot.next_shred_index as i64,
+                    i64
+                ),
+                (
+                    "get_leader_schedule_time",
+                    stats.get_leader_schedule_elapsed,
+                    i64
+                ),
+                ("serialize_shreds_time", stats.serialize_elapsed, i64),
+                ("gen_data_time", stats.gen_data_elapsed, i64),
+                ("gen_coding_time", stats.gen_coding_elapsed, i64),
+                ("sign_coding_time", stats.sign_coding_elapsed, i64),
+                ("coding_send_time", stats.coding_send_elapsed, i64),
+            );
+        } else {
+            datapoint_info!(
+                "broadcast-process-shreds-stats",
+                ("slot", unfinished_slot.slot as i64, i64),
+                ("shredding_time", stats.shredding_elapsed, i64),
+                ("receive_time", stats.receive_elapsed, i64),
+                (
+                    "num_data_shreds",
+                    unfinished_slot.next_shred_index as i64,
+                    i64
+                ),
+                (
+                    "slot_broadcast_time",
+                    self.slot_broadcast_start.unwrap().elapsed().as_micros() as i64,
+                    i64
+                ),
+                (
+                    "get_leader_schedule_time",
+                    stats.get_leader_schedule_elapsed,
+                    i64
+                ),
+                ("serialize_shreds_time", stats.serialize_elapsed, i64),
+                ("gen_data_time", stats.gen_data_elapsed, i64),
+                ("gen_coding_time", stats.gen_coding_elapsed, i64),
+                ("sign_coding_time", stats.sign_coding_elapsed, i64),
+                ("coding_send_time", stats.coding_send_elapsed, i64),
+            );
+        }
         self.process_shreds_stats.reset();
     }
 }

core/src/cost_update_service.rs

@@ -0,0 +1,291 @@
+//! this service receives instruction ExecuteTimings from replay_stage,
+//! update cost_model which is shared with banking_stage to optimize
+//! packing transactions into block; it also triggers persisting cost
+//! table to blockstore.
+
+use solana_ledger::blockstore::Blockstore;
+use solana_measure::measure::Measure;
+use solana_runtime::{bank::ExecuteTimings, cost_model::CostModel};
+use solana_sdk::timing::timestamp;
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        mpsc::Receiver,
+        Arc, RwLock,
+    },
+    thread::{self, Builder, JoinHandle},
+    time::Duration,
+};
+
+#[derive(Default)]
+pub struct CostUpdateServiceTiming {
+    last_print: u64,
+    update_cost_model_count: u64,
+    update_cost_model_elapsed: u64,
+    persist_cost_table_elapsed: u64,
+}
+
+impl CostUpdateServiceTiming {
+    fn update(
+        &mut self,
+        update_cost_model_count: u64,
+        update_cost_model_elapsed: u64,
+        persist_cost_table_elapsed: u64,
+    ) {
+        self.update_cost_model_count += update_cost_model_count;
+        self.update_cost_model_elapsed += update_cost_model_elapsed;
+        self.persist_cost_table_elapsed += persist_cost_table_elapsed;
+
+        let now = timestamp();
+        let elapsed_ms = now - self.last_print;
+        if elapsed_ms > 1000 {
+            datapoint_info!(
+                "cost-update-service-stats",
+                ("total_elapsed_us", elapsed_ms * 1000, i64),
+                (
+                    "update_cost_model_count",
+                    self.update_cost_model_count as i64,
+                    i64
+                ),
+                (
+                    "update_cost_model_elapsed",
+                    self.update_cost_model_elapsed as i64,
+                    i64
+                ),
+                (
+                    "persist_cost_table_elapsed",
+                    self.persist_cost_table_elapsed as i64,
+                    i64
+                ),
+            );
+
+            *self = CostUpdateServiceTiming::default();
+            self.last_print = now;
+        }
+    }
+}
+
+pub type CostUpdateReceiver = Receiver<ExecuteTimings>;
+
+pub struct CostUpdateService {
+    thread_hdl: JoinHandle<()>,
+}
+
+impl CostUpdateService {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(
+        exit: Arc<AtomicBool>,
+        blockstore: Arc<Blockstore>,
+        cost_model: Arc<RwLock<CostModel>>,
+        cost_update_receiver: CostUpdateReceiver,
+    ) -> Self {
+        let thread_hdl = Builder::new()
+            .name("solana-cost-update-service".to_string())
+            .spawn(move || {
+                Self::service_loop(exit, blockstore, cost_model, cost_update_receiver);
+            })
+            .unwrap();
+
+        Self { thread_hdl }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.thread_hdl.join()
+    }
+
+    fn service_loop(
+        exit: Arc<AtomicBool>,
+        blockstore: Arc<Blockstore>,
+        cost_model: Arc<RwLock<CostModel>>,
+        cost_update_receiver: CostUpdateReceiver,
+    ) {
+        let mut cost_update_service_timing = CostUpdateServiceTiming::default();
+        let mut dirty: bool;
+        let mut update_count: u64;
+        let wait_timer = Duration::from_millis(100);
+
+        loop {
+            if exit.load(Ordering::Relaxed) {
+                break;
+            }
+
+            dirty = false;
+            update_count = 0_u64;
+            let mut update_cost_model_time = Measure::start("update_cost_model_time");
+            for cost_update in cost_update_receiver.try_iter() {
+                dirty |= Self::update_cost_model(&cost_model, &cost_update);
+                update_count += 1;
+            }
+            update_cost_model_time.stop();
+
+            let mut persist_cost_table_time = Measure::start("persist_cost_table_time");
+            if dirty {
+                Self::persist_cost_table(&blockstore, &cost_model);
+            }
+            persist_cost_table_time.stop();
+
+            cost_update_service_timing.update(
+                update_count,
+                update_cost_model_time.as_us(),
+                persist_cost_table_time.as_us(),
+            );
+
+            thread::sleep(wait_timer);
+        }
+    }
+
+    fn update_cost_model(cost_model: &RwLock<CostModel>, execute_timings: &ExecuteTimings) -> bool {
+        let mut dirty = false;
+        {
+            let mut cost_model_mutable = cost_model.write().unwrap();
+            for (program_id, timing) in &execute_timings.details.per_program_timings {
+                if timing.count < 1 {
+                    continue;
+                }
+                let units = timing.accumulated_units / timing.count as u64;
+                match cost_model_mutable.upsert_instruction_cost(program_id, units) {
+                    Ok(c) => {
+                        debug!(
+                            "after replayed into bank, instruction {:?} has averaged cost {}",
+                            program_id, c
+                        );
+                        dirty = true;
+                    }
+                    Err(err) => {
+                        debug!(
+                        "after replayed into bank, instruction {:?} failed to update cost, err: {}",
+                        program_id, err
+                    );
+                    }
+                }
+            }
+        }
+        debug!(
+           "after replayed into bank, updated cost model instruction cost table, current values: {:?}",
+           cost_model.read().unwrap().get_instruction_cost_table()
+        );
+        dirty
+    }
+
+    fn persist_cost_table(blockstore: &Blockstore, cost_model: &RwLock<CostModel>) {
+        let cost_model_read = cost_model.read().unwrap();
+        let cost_table = cost_model_read.get_instruction_cost_table();
+        let db_records = blockstore.read_program_costs().expect("read programs");
+
+        // delete records from blockstore if they are no longer in cost_table
+        db_records.iter().for_each(|(pubkey, _)| {
+            if cost_table.get(pubkey).is_none() {
+                blockstore
+                    .delete_program_cost(pubkey)
+                    .expect("delete old program");
+            }
+        });
+
+        for (key, cost) in cost_table.iter() {
+            blockstore
+                .write_program_cost(key, cost)
+                .expect("persist program costs to blockstore");
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_runtime::message_processor::ProgramTiming;
+    use solana_sdk::pubkey::Pubkey;
+
+    #[test]
+    fn test_update_cost_model_with_empty_execute_timings() {
+        let cost_model = Arc::new(RwLock::new(CostModel::default()));
+        let empty_execute_timings = ExecuteTimings::default();
+        CostUpdateService::update_cost_model(&cost_model, &empty_execute_timings);
+
+        assert_eq!(
+            0,
+            cost_model
+                .read()
+                .unwrap()
+                .get_instruction_cost_table()
+                .len()
+        );
+    }
+
+    #[test]
+    fn test_update_cost_model_with_execute_timings() {
+        let cost_model = Arc::new(RwLock::new(CostModel::default()));
+        let mut execute_timings = ExecuteTimings::default();
+
+        let program_key_1 = Pubkey::new_unique();
+        let mut expected_cost: u64;
+
+        // add new program
+        {
+            let accumulated_us: u64 = 1000;
+            let accumulated_units: u64 = 100;
+            let count: u32 = 10;
+            expected_cost = accumulated_units / count as u64;
+
+            execute_timings.details.per_program_timings.insert(
+                program_key_1,
+                ProgramTiming {
+                    accumulated_us,
+                    accumulated_units,
+                    count,
+                },
+            );
+            CostUpdateService::update_cost_model(&cost_model, &execute_timings);
+            assert_eq!(
+                1,
+                cost_model
+                    .read()
+                    .unwrap()
+                    .get_instruction_cost_table()
+                    .len()
+            );
+            assert_eq!(
+                Some(&expected_cost),
+                cost_model
+                    .read()
+                    .unwrap()
+                    .get_instruction_cost_table()
+                    .get(&program_key_1)
+            );
+        }
+
+        // update program
+        {
+            let accumulated_us: u64 = 2000;
+            let accumulated_units: u64 = 200;
+            let count: u32 = 10;
+            // to expect new cost is Average(new_value, existing_value)
+            expected_cost = ((accumulated_units / count as u64) + expected_cost) / 2;
+
+            execute_timings.details.per_program_timings.insert(
+                program_key_1,
+                ProgramTiming {
+                    accumulated_us,
+                    accumulated_units,
+                    count,
+                },
+            );
+            CostUpdateService::update_cost_model(&cost_model, &execute_timings);
+            assert_eq!(
+                1,
+                cost_model
+                    .read()
+                    .unwrap()
+                    .get_instruction_cost_table()
+                    .len()
+            );
+            assert_eq!(
+                Some(&expected_cost),
+                cost_model
+                    .read()
+                    .unwrap()
+                    .get_instruction_cost_table()
+                    .get(&program_key_1)
+            );
+        }
+    }
+}

core/src/lib.rs

@@ -19,6 +19,7 @@ pub mod cluster_slots_service;
 pub mod commitment_service;
 pub mod completed_data_sets_service;
 pub mod consensus;
+pub mod cost_update_service;
 pub mod fetch_stage;
 pub mod fork_choice;
 pub mod gen_keys;
@@ -46,6 +47,7 @@ pub mod sigverify;
 pub mod sigverify_shreds;
 pub mod sigverify_stage;
 pub mod snapshot_packager_service;
+pub mod system_monitor_service;
 pub mod test_validator;
 pub mod tpu;
 pub mod tree_diff;

core/src/progress_map.rs

@@ -114,6 +114,43 @@ impl ReplaySlotStats {
                 i64
             ),
         );
+
+        let mut per_pubkey_timings: Vec<_> = self
+            .execute_timings
+            .details
+            .per_program_timings
+            .iter()
+            .collect();
+        per_pubkey_timings.sort_by(|a, b| b.1.accumulated_us.cmp(&a.1.accumulated_us));
+        let (total_us, total_units, total_count) =
+            per_pubkey_timings
+                .iter()
+                .fold((0, 0, 0), |(sum_us, sum_units, sum_count), a| {
+                    (
+                        sum_us + a.1.accumulated_us,
+                        sum_units + a.1.accumulated_units,
+                        sum_count + a.1.count,
+                    )
+                });
+
+        for (pubkey, time) in per_pubkey_timings.iter().take(5) {
+            datapoint_info!(
+                "per_program_timings",
+                ("slot", slot as i64, i64),
+                ("pubkey", pubkey.to_string(), String),
+                ("execute_us", time.accumulated_us, i64),
+                ("accumulated_units", time.accumulated_units, i64),
+                ("count", time.count, i64)
+            );
+        }
+        datapoint_info!(
+            "per_program_timings",
+            ("slot", slot as i64, i64),
+            ("pubkey", "all", String),
+            ("execute_us", total_us, i64),
+            ("accumulated_units", total_units, i64),
+            ("count", total_count, i64)
+        );
     }
 }
 

core/src/replay_stage.rs

@@ -18,7 +18,6 @@ use crate::{
     latest_validator_votes_for_frozen_banks::LatestValidatorVotesForFrozenBanks,
     progress_map::{ForkProgress, ProgressMap, PropagatedStats},
     repair_service::DuplicateSlotsResetReceiver,
-    result::Result,
     rewards_recorder_service::RewardsRecorderSender,
     unfrozen_gossip_verified_vote_hashes::UnfrozenGossipVerifiedVoteHashes,
     voting_service::VoteOp,
@@ -42,7 +41,7 @@ use solana_rpc::{
 };
 use solana_runtime::{
     accounts_background_service::AbsRequestSender,
-    bank::{Bank, NewBankOptions},
+    bank::{Bank, ExecuteTimings, NewBankOptions},
     bank_forks::BankForks,
     commitment::BlockCommitmentCache,
     vote_sender_types::ReplayVoteSender,
@@ -281,7 +280,7 @@ impl ReplayTiming {
                     "process_duplicate_slots_elapsed",
                     self.process_duplicate_slots_elapsed as i64,
                     i64
-                )
+                ),
             );
 
             *self = ReplayTiming::default();
@@ -291,7 +290,7 @@ impl ReplayTiming {
 }
 
 pub struct ReplayStage {
-    t_replay: JoinHandle<Result<()>>,
+    t_replay: JoinHandle<()>,
     commitment_service: AggregateCommitmentService,
 }
 
@@ -315,6 +314,7 @@ impl ReplayStage {
         gossip_verified_vote_hash_receiver: GossipVerifiedVoteHashReceiver,
         cluster_slots_update_sender: ClusterSlotsUpdateSender,
         voting_sender: Sender<VoteOp>,
+        cost_update_sender: Sender<ExecuteTimings>,
     ) -> Self {
         let ReplayStageConfig {
             my_pubkey,
@@ -412,6 +412,7 @@ impl ReplayStage {
                         &mut unfrozen_gossip_verified_vote_hashes,
                         &mut latest_validator_votes_for_frozen_banks,
                         &cluster_slots_update_sender,
+                        &cost_update_sender,
                     );
                     replay_active_banks_time.stop();
 
@@ -742,7 +743,6 @@ impl ReplayStage {
                         process_duplicate_slots_time.as_us(),
                     );
                 }
-                Ok(())
             })
             .unwrap();
 
@@ -1690,9 +1690,11 @@ impl ReplayStage {
         unfrozen_gossip_verified_vote_hashes: &mut UnfrozenGossipVerifiedVoteHashes,
         latest_validator_votes_for_frozen_banks: &mut LatestValidatorVotesForFrozenBanks,
         cluster_slots_update_sender: &ClusterSlotsUpdateSender,
+        cost_update_sender: &Sender<ExecuteTimings>,
     ) -> bool {
         let mut did_complete_bank = false;
         let mut tx_count = 0;
+        let mut execute_timings = ExecuteTimings::default();
         let active_banks = bank_forks.read().unwrap().active_banks();
         trace!("active banks {:?}", active_banks);
 
@@ -1763,6 +1765,12 @@ impl ReplayStage {
             }
             assert_eq!(*bank_slot, bank.slot());
             if bank.is_complete() {
+                execute_timings.accumulate(&bank_progress.replay_stats.execute_timings);
+                debug!("bank {} is completed replay from blockstore, contribute to update cost with {:?}",
+                       bank.slot(),
+                       bank_progress.replay_stats.execute_timings
+                       );
+
                 bank_progress.replay_stats.report_stats(
                     bank.slot(),
                     bank_progress.replay_progress.num_entries,
@@ -1824,6 +1832,14 @@ impl ReplayStage {
                 );
             }
         }
+
+        // send accumulated excute-timings to cost_update_service
+        if !execute_timings.details.per_program_timings.is_empty() {
+            cost_update_sender
+                .send(execute_timings)
+                .unwrap_or_else(|err| warn!("cost_update_sender failed: {:?}", err));
+        }
+
         inc_new_counter_info!("replay_stage-replay_transactions", tx_count);
         did_complete_bank
     }
@@ -4929,7 +4945,6 @@ mod tests {
         );
         assert_eq!(tower.last_voted_slot().unwrap(), 1);
     }
-
     fn run_compute_and_select_forks(
         bank_forks: &RwLock<BankForks>,
         progress: &mut ProgressMap,

core/src/retransmit_stage.rs

@@ -495,7 +495,7 @@ mod tests {
             ..ProcessOptions::default()
         };
         let (bank_forks, cached_leader_schedule) =
-            process_blockstore(&genesis_config, &blockstore, Vec::new(), opts, None).unwrap();
+            process_blockstore(&genesis_config, &blockstore, Vec::new(), opts, None, None).unwrap();
         let leader_schedule_cache = Arc::new(cached_leader_schedule);
         let bank_forks = Arc::new(RwLock::new(bank_forks));
 

core/src/sigverify_stage.rs

@@ -8,13 +8,13 @@
 use crate::sigverify;
 use crossbeam_channel::{SendError, Sender as CrossbeamSender};
 use solana_measure::measure::Measure;
-use solana_metrics::datapoint_debug;
 use solana_perf::packet::Packets;
 use solana_sdk::timing;
 use solana_streamer::streamer::{self, PacketReceiver, StreamerError};
 use std::collections::HashMap;
 use std::sync::mpsc::{Receiver, RecvTimeoutError};
 use std::thread::{self, Builder, JoinHandle};
+use std::time::Instant;
 use thiserror::Error;
 
 const MAX_SIGVERIFY_BATCH: usize = 10_000;
@@ -41,6 +41,82 @@ pub trait SigVerifier {
 #[derive(Default, Clone)]
 pub struct DisabledSigVerifier {}
 
+#[derive(Default)]
+struct SigVerifierStats {
+    recv_batches_us_hist: histogram::Histogram, // time to call recv_batch
+    verify_batches_pp_us_hist: histogram::Histogram, // per-packet time to call verify_batch
+    batches_hist: histogram::Histogram,         // number of Packets structures per verify call
+    packets_hist: histogram::Histogram,         // number of packets per verify call
+    total_batches: usize,
+    total_packets: usize,
+}
+
+impl SigVerifierStats {
+    fn report(&self) {
+        datapoint_info!(
+            "sigverify_stage-total_verify_time",
+            (
+                "recv_batches_us_90pct",
+                self.recv_batches_us_hist.percentile(90.0).unwrap_or(0),
+                i64
+            ),
+            (
+                "recv_batches_us_min",
+                self.recv_batches_us_hist.minimum().unwrap_or(0),
+                i64
+            ),
+            (
+                "recv_batches_us_max",
+                self.recv_batches_us_hist.maximum().unwrap_or(0),
+                i64
+            ),
+            (
+                "recv_batches_us_mean",
+                self.recv_batches_us_hist.mean().unwrap_or(0),
+                i64
+            ),
+            (
+                "verify_batches_pp_us_90pct",
+                self.verify_batches_pp_us_hist.percentile(90.0).unwrap_or(0),
+                i64
+            ),
+            (
+                "verify_batches_pp_us_min",
+                self.verify_batches_pp_us_hist.minimum().unwrap_or(0),
+                i64
+            ),
+            (
+                "verify_batches_pp_us_max",
+                self.verify_batches_pp_us_hist.maximum().unwrap_or(0),
+                i64
+            ),
+            (
+                "verify_batches_pp_us_mean",
+                self.verify_batches_pp_us_hist.mean().unwrap_or(0),
+                i64
+            ),
+            (
+                "batches_90pct",
+                self.batches_hist.percentile(90.0).unwrap_or(0),
+                i64
+            ),
+            ("batches_min", self.batches_hist.minimum().unwrap_or(0), i64),
+            ("batches_max", self.batches_hist.maximum().unwrap_or(0), i64),
+            ("batches_mean", self.batches_hist.mean().unwrap_or(0), i64),
+            (
+                "packets_90pct",
+                self.packets_hist.percentile(90.0).unwrap_or(0),
+                i64
+            ),
+            ("packets_min", self.packets_hist.minimum().unwrap_or(0), i64),
+            ("packets_max", self.packets_hist.maximum().unwrap_or(0), i64),
+            ("packets_mean", self.packets_hist.mean().unwrap_or(0), i64),
+            ("total_batches", self.total_batches, i64),
+            ("total_packets", self.total_packets, i64),
+        );
+    }
+}
+
 impl SigVerifier for DisabledSigVerifier {
     fn verify_batch(&self, mut batch: Vec<Packets>) -> Vec<Packets> {
         sigverify::ed25519_verify_disabled(&mut batch);
@@ -92,6 +168,7 @@ impl SigVerifyStage {
         recvr: &PacketReceiver,
         sendr: &CrossbeamSender<Vec<Packets>>,
         verifier: &T,
+        stats: &mut SigVerifierStats,
     ) -> Result<()> {
         let (mut batches, len, recv_time) = streamer::recv_batch(recvr)?;
 
@@ -121,6 +198,19 @@ impl SigVerifyStage {
             ("recv_time", recv_time, i64),
         );
 
+        stats
+            .recv_batches_us_hist
+            .increment(recv_time as u64)
+            .unwrap();
+        stats
+            .verify_batches_pp_us_hist
+            .increment(verify_batch_time.as_us() / (len as u64))
+            .unwrap();
+        stats.batches_hist.increment(batches_len as u64).unwrap();
+        stats.packets_hist.increment(len as u64).unwrap();
+        stats.total_batches += batches_len;
+        stats.total_packets += len;
+
         Ok(())
     }
 
@@ -130,10 +220,14 @@ impl SigVerifyStage {
         verifier: &T,
     ) -> JoinHandle<()> {
         let verifier = verifier.clone();
+        let mut stats = SigVerifierStats::default();
+        let mut last_print = Instant::now();
         Builder::new()
             .name("solana-verifier".to_string())
             .spawn(move || loop {
-                if let Err(e) = Self::verifier(&packet_receiver, &verified_sender, &verifier) {
+                if let Err(e) =
+                    Self::verifier(&packet_receiver, &verified_sender, &verifier, &mut stats)
+                {
                     match e {
                         SigVerifyServiceError::Streamer(StreamerError::RecvTimeout(
                             RecvTimeoutError::Disconnected,
@@ -147,6 +241,11 @@ impl SigVerifyStage {
                         _ => error!("{:?}", e),
                     }
                 }
+                if last_print.elapsed().as_secs() > 2 {
+                    stats.report();
+                    stats = SigVerifierStats::default();
+                    last_print = Instant::now();
+                }
             })
             .unwrap()
     }

core/src/system_monitor_service.rs

@@ -0,0 +1,227 @@
+use std::{
+    collections::HashMap,
+    io::BufRead,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    thread::{self, sleep, Builder, JoinHandle},
+    time::{Duration, Instant},
+};
+
+#[cfg(target_os = "linux")]
+use std::{fs::File, io::BufReader, path::Path};
+
+const SAMPLE_INTERVAL: Duration = Duration::from_secs(60);
+const SLEEP_INTERVAL: Duration = Duration::from_millis(500);
+
+#[cfg(target_os = "linux")]
+const PROC_NET_SNMP_PATH: &str = "/proc/net/snmp";
+
+pub struct SystemMonitorService {
+    thread_hdl: JoinHandle<()>,
+}
+
+#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+struct UdpStats {
+    in_datagrams: usize,
+    no_ports: usize,
+    in_errors: usize,
+    out_datagrams: usize,
+    rcvbuf_errors: usize,
+    sndbuf_errors: usize,
+    in_csum_errors: usize,
+    ignored_multi: usize,
+}
+
+#[cfg(target_os = "linux")]
+fn read_udp_stats(file_path: impl AsRef<Path>) -> Result<UdpStats, String> {
+    let file = File::open(file_path).map_err(|e| e.to_string())?;
+    let mut reader = BufReader::new(file);
+    parse_udp_stats(&mut reader)
+}
+
+#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
+fn parse_udp_stats(reader: &mut impl BufRead) -> Result<UdpStats, String> {
+    let mut udp_lines = Vec::default();
+    for line in reader.lines() {
+        let line = line.map_err(|e| e.to_string())?;
+        if line.starts_with("Udp:") {
+            udp_lines.push(line);
+            if udp_lines.len() == 2 {
+                break;
+            }
+        }
+    }
+    if udp_lines.len() != 2 {
+        return Err(format!(
+            "parse error, expected 2 lines, num lines: {}",
+            udp_lines.len()
+        ));
+    }
+
+    let pairs: Vec<_> = udp_lines[0]
+        .split_ascii_whitespace()
+        .zip(udp_lines[1].split_ascii_whitespace())
+        .collect();
+    let udp_stats: HashMap<String, usize> = pairs[1..]
+        .iter()
+        .map(|(label, val)| (label.to_string(), val.parse::<usize>().unwrap()))
+        .collect();
+
+    let stats = UdpStats {
+        in_datagrams: *udp_stats.get("InDatagrams").unwrap_or(&0),
+        no_ports: *udp_stats.get("NoPorts").unwrap_or(&0),
+        in_errors: *udp_stats.get("InErrors").unwrap_or(&0),
+        out_datagrams: *udp_stats.get("OutDatagrams").unwrap_or(&0),
+        rcvbuf_errors: *udp_stats.get("RcvbufErrors").unwrap_or(&0),
+        sndbuf_errors: *udp_stats.get("SndbufErrors").unwrap_or(&0),
+        in_csum_errors: *udp_stats.get("InCsumErrors").unwrap_or(&0),
+        ignored_multi: *udp_stats.get("IgnoredMulti").unwrap_or(&0),
+    };
+
+    Ok(stats)
+}
+
+#[cfg(target_os = "linux")]
+pub fn verify_udp_stats_access() -> Result<(), String> {
+    read_udp_stats(PROC_NET_SNMP_PATH)?;
+    Ok(())
+}
+
+#[cfg(not(target_os = "linux"))]
+pub fn verify_udp_stats_access() -> Result<(), String> {
+    Ok(())
+}
+
+impl SystemMonitorService {
+    pub fn new(exit: Arc<AtomicBool>) -> Self {
+        info!("Starting SystemMonitorService");
+        let thread_hdl = Builder::new()
+            .name("system-monitor".to_string())
+            .spawn(move || {
+                Self::run(exit);
+            })
+            .unwrap();
+
+        Self { thread_hdl }
+    }
+
+    #[cfg(target_os = "linux")]
+    fn process_udp_stats(udp_stats: &mut Option<UdpStats>) {
+        match read_udp_stats(PROC_NET_SNMP_PATH) {
+            Ok(new_stats) => {
+                if let Some(old_stats) = udp_stats {
+                    SystemMonitorService::report_udp_stats(old_stats, &new_stats);
+                }
+                *udp_stats = Some(new_stats);
+            }
+            Err(e) => warn!("read_udp_stats: {}", e),
+        }
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    fn process_udp_stats(_udp_stats: &mut Option<UdpStats>) {}
+
+    #[cfg(target_os = "linux")]
+    fn report_udp_stats(old_stats: &UdpStats, new_stats: &UdpStats) {
+        datapoint_info!(
+            "net-stats",
+            (
+                "in_datagrams_delta",
+                new_stats.in_datagrams - old_stats.in_datagrams,
+                i64
+            ),
+            (
+                "no_ports_delta",
+                new_stats.no_ports - old_stats.no_ports,
+                i64
+            ),
+            (
+                "in_errors_delta",
+                new_stats.in_errors - old_stats.in_errors,
+                i64
+            ),
+            (
+                "out_datagrams_delta",
+                new_stats.out_datagrams - old_stats.out_datagrams,
+                i64
+            ),
+            (
+                "rcvbuf_errors_delta",
+                new_stats.rcvbuf_errors - old_stats.rcvbuf_errors,
+                i64
+            ),
+            (
+                "sndbuf_errors_delta",
+                new_stats.sndbuf_errors - old_stats.sndbuf_errors,
+                i64
+            ),
+            (
+                "in_csum_errors_delta",
+                new_stats.in_csum_errors - old_stats.in_csum_errors,
+                i64
+            ),
+            (
+                "ignored_multi_delta",
+                new_stats.ignored_multi - old_stats.ignored_multi,
+                i64
+            ),
+            ("in_errors", new_stats.in_errors, i64),
+            ("rcvbuf_errors", new_stats.rcvbuf_errors, i64),
+            ("sndbuf_errors", new_stats.sndbuf_errors, i64),
+        );
+    }
+
+    pub fn run(exit: Arc<AtomicBool>) {
+        let mut udp_stats = None;
+
+        let mut now = Instant::now();
+        loop {
+            if exit.load(Ordering::Relaxed) {
+                break;
+            }
+
+            if now.elapsed() >= SAMPLE_INTERVAL {
+                now = Instant::now();
+
+                SystemMonitorService::process_udp_stats(&mut udp_stats);
+            }
+
+            sleep(SLEEP_INTERVAL);
+        }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.thread_hdl.join()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_udp_stats() {
+        let mut mock_snmp =
+b"Ip: Forwarding DefaultTTL InReceives InHdrErrors InAddrErrors ForwDatagrams InUnknownProtos InDiscards InDelivers OutRequests OutDiscards OutNoRoutes ReasmTimeout ReasmReqds ReasmOKs ReasmFails FragOKs FragFails FragCreates
+Ip: 1 64 357 0 2 0 0 0 355 315 0 6 0 0 0 0 0 0 0
+Icmp: InMsgs InErrors InCsumErrors InDestUnreachs InTimeExcds InParmProbs InSrcQuenchs InRedirects InEchos InEchoReps InTimestamps InTimestampReps InAddrMasks InAddrMaskReps OutMsgs OutErrors OutDestUnreachs OutTimeExcds OutParmProbs OutSrcQuenchs OutRedirects OutEchos OutEchoReps OutTimestamps OutTimestampReps OutAddrMasks OutAddrMaskReps
+Icmp: 3 0 0 3 0 0 0 0 0 0 0 0 0 0 7 0 7 0 0 0 0 0 0 0 0 0 0
+IcmpMsg: InType3 OutType3
+IcmpMsg: 3 7
+Tcp: RtoAlgorithm RtoMin RtoMax MaxConn ActiveOpens PassiveOpens AttemptFails EstabResets CurrEstab InSegs OutSegs RetransSegs InErrs OutRsts InCsumErrors
+Tcp: 1 200 120000 -1 29 1 0 0 5 318 279 0 0 4 0
+Udp: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors InCsumErrors IgnoredMulti
+Udp: 27 7 0 30 0 0 0 0
+UdpLite: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors InCsumErrors IgnoredMulti
+UdpLite: 0 0 0 0 0 0 0 0" as &[u8];
+        let stats = parse_udp_stats(&mut mock_snmp).unwrap();
+        assert_eq!(stats.out_datagrams, 30);
+        assert_eq!(stats.no_ports, 7);
+
+        let mut mock_snmp = b"unexpected data" as &[u8];
+        let stats = parse_udp_stats(&mut mock_snmp);
+        assert!(stats.is_err());
+    }
+}

core/src/tpu.rs

@@ -22,6 +22,7 @@ use solana_rpc::{
 };
 use solana_runtime::{
     bank_forks::BankForks,
+    cost_model::CostModel,
     vote_sender_types::{ReplayVoteReceiver, ReplayVoteSender},
 };
 use std::{
@@ -71,6 +72,7 @@ impl Tpu {
         bank_notification_sender: Option<BankNotificationSender>,
         tpu_coalesce_ms: u64,
         cluster_confirmed_slot_sender: GossipDuplicateConfirmedSlotsSender,
+        cost_model: &Arc<RwLock<CostModel>>,
     ) -> Self {
         let (packet_sender, packet_receiver) = channel();
         let (vote_packet_sender, vote_packet_receiver) = channel();
@@ -128,6 +130,7 @@ impl Tpu {
             verified_gossip_vote_packets_receiver,
             transaction_status_sender,
             replay_vote_sender,
+            cost_model.clone(),
         );
 
         let broadcast_stage = broadcast_type.new_broadcast_stage(

core/src/tvu.rs

@@ -12,6 +12,7 @@ use crate::{
     cluster_slots::ClusterSlots,
     completed_data_sets_service::CompletedDataSetsSender,
     consensus::Tower,
+    cost_update_service::CostUpdateService,
     ledger_cleanup_service::LedgerCleanupService,
     replay_stage::{ReplayStage, ReplayStageConfig},
     retransmit_stage::RetransmitStage,
@@ -38,8 +39,10 @@ use solana_runtime::{
         AbsRequestHandler, AbsRequestSender, AccountsBackgroundService, SnapshotRequestHandler,
     },
     accounts_db::AccountShrinkThreshold,
+    bank::ExecuteTimings,
     bank_forks::{BankForks, SnapshotConfig},
     commitment::BlockCommitmentCache,
+    cost_model::CostModel,
     vote_sender_types::ReplayVoteSender,
 };
 use solana_sdk::{
@@ -52,7 +55,7 @@ use std::{
     net::UdpSocket,
     sync::{
         atomic::AtomicBool,
-        mpsc::{channel, Receiver},
+        mpsc::{channel, Receiver, Sender},
         Arc, Mutex, RwLock,
     },
     thread,
@@ -67,6 +70,7 @@ pub struct Tvu {
     accounts_background_service: AccountsBackgroundService,
     accounts_hash_verifier: AccountsHashVerifier,
     voting_service: VotingService,
+    cost_update_service: CostUpdateService,
 }
 
 pub struct Sockets {
@@ -131,6 +135,7 @@ impl Tvu {
         gossip_confirmed_slots_receiver: GossipDuplicateConfirmedSlotsReceiver,
         tvu_config: TvuConfig,
         max_slots: &Arc<MaxSlots>,
+        cost_model: &Arc<RwLock<CostModel>>,
     ) -> Self {
         let keypair: Arc<Keypair> = cluster_info.keypair.clone();
 
@@ -285,6 +290,17 @@ impl Tvu {
             bank_forks.clone(),
         );
 
+        let (cost_update_sender, cost_update_receiver): (
+            Sender<ExecuteTimings>,
+            Receiver<ExecuteTimings>,
+        ) = channel();
+        let cost_update_service = CostUpdateService::new(
+            exit.clone(),
+            blockstore.clone(),
+            cost_model.clone(),
+            cost_update_receiver,
+        );
+
         let replay_stage = ReplayStage::new(
             replay_stage_config,
             blockstore.clone(),
@@ -303,6 +319,7 @@ impl Tvu {
             gossip_verified_vote_hash_receiver,
             cluster_slots_update_sender,
             voting_sender,
+            cost_update_sender,
         );
 
         let ledger_cleanup_service = tvu_config.max_ledger_shreds.map(|max_ledger_shreds| {
@@ -334,6 +351,7 @@ impl Tvu {
             accounts_background_service,
             accounts_hash_verifier,
             voting_service,
+            cost_update_service,
         }
     }
 
@@ -348,6 +366,7 @@ impl Tvu {
         self.replay_stage.join()?;
         self.accounts_hash_verifier.join()?;
         self.voting_service.join()?;
+        self.cost_update_service.join()?;
         Ok(())
     }
 }
@@ -455,6 +474,7 @@ pub mod tests {
             gossip_confirmed_slots_receiver,
             TvuConfig::default(),
             &Arc::new(MaxSlots::default()),
+            &Arc::new(RwLock::new(CostModel::default())),
         );
         exit.store(true, Ordering::Relaxed);
         tvu.join().unwrap();

core/src/validator.rs

@@ -1,5 +1,6 @@
 //! The `validator` module hosts all the validator microservices.
 
+pub use solana_perf::report_target_features;
 use {
     crate::{
         broadcast_stage::BroadcastStageType,
@@ -13,11 +14,13 @@ use {
         serve_repair_service::ServeRepairService,
         sigverify,
         snapshot_packager_service::{PendingSnapshotPackage, SnapshotPackagerService},
+        system_monitor_service::{verify_udp_stats_access, SystemMonitorService},
         tpu::{Tpu, DEFAULT_TPU_COALESCE_MS},
         tvu::{Sockets, Tvu, TvuConfig},
     },
     crossbeam_channel::{bounded, unbounded},
     rand::{thread_rng, Rng},
+    solana_accountsdb_plugin_manager::accountsdb_plugin_service::AccountsDbPluginService,
     solana_gossip::{
         cluster_info::{
             ClusterInfo, Node, DEFAULT_CONTACT_DEBUG_INTERVAL_MILLIS,
@@ -42,6 +45,7 @@ use {
         poh_recorder::{PohRecorder, GRACE_TICKS_FACTOR, MAX_GRACE_SLOTS},
         poh_service::{self, PohService},
     },
+    solana_rpc::send_transaction_service,
     solana_rpc::{
         max_slots::MaxSlots,
         optimistically_confirmed_bank_tracker::{
@@ -57,9 +61,11 @@ use {
     solana_runtime::{
         accounts_db::AccountShrinkThreshold,
         accounts_index::AccountSecondaryIndexes,
+        accounts_update_notifier_interface::AccountsUpdateNotifier,
         bank::Bank,
         bank_forks::{BankForks, SnapshotConfig},
         commitment::BlockCommitmentCache,
+        cost_model::CostModel,
         hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
     },
     solana_sdk::{
@@ -103,6 +109,7 @@ pub struct ValidatorConfig {
     pub account_paths: Vec<PathBuf>,
     pub account_shrink_paths: Option<Vec<PathBuf>>,
     pub rpc_config: JsonRpcConfig,
+    pub accountsdb_plugin_config_files: Option<Vec<PathBuf>>,
     pub rpc_addrs: Option<(SocketAddr, SocketAddr)>, // (JsonRpc, JsonRpcPubSub)
     pub pubsub_config: PubSubConfig,
     pub snapshot_config: Option<SnapshotConfig>,
@@ -132,8 +139,7 @@ pub struct ValidatorConfig {
     pub contact_debug_interval: u64,
     pub contact_save_interval: u64,
     pub bpf_jit: bool,
-    pub send_transaction_retry_ms: u64,
-    pub send_transaction_leader_forward_count: u64,
+    pub send_transaction_service_config: send_transaction_service::Config,
     pub no_poh_speed_test: bool,
     pub poh_pinned_cpu_core: usize,
     pub poh_hashes_per_batch: u64,
@@ -162,6 +168,7 @@ impl Default for ValidatorConfig {
             account_paths: Vec::new(),
             account_shrink_paths: None,
             rpc_config: JsonRpcConfig::default(),
+            accountsdb_plugin_config_files: None,
             rpc_addrs: None,
             pubsub_config: PubSubConfig::default(),
             snapshot_config: None,
@@ -190,8 +197,7 @@ impl Default for ValidatorConfig {
             contact_debug_interval: DEFAULT_CONTACT_DEBUG_INTERVAL_MILLIS,
             contact_save_interval: DEFAULT_CONTACT_SAVE_INTERVAL_MILLIS,
             bpf_jit: false,
-            send_transaction_retry_ms: 2000,
-            send_transaction_leader_forward_count: 2,
+            send_transaction_service_config: send_transaction_service::Config::default(),
             no_poh_speed_test: true,
             poh_pinned_cpu_core: poh_service::DEFAULT_PINNED_CPU_CORE,
             poh_hashes_per_batch: poh_service::DEFAULT_HASHES_PER_BATCH,
@@ -256,6 +262,7 @@ pub struct Validator {
     transaction_status_service: Option<TransactionStatusService>,
     rewards_recorder_service: Option<RewardsRecorderService>,
     cache_block_meta_service: Option<CacheBlockMetaService>,
+    system_monitor_service: Option<SystemMonitorService>,
     sample_performance_service: Option<SamplePerformanceService>,
     gossip_service: GossipService,
     serve_repair_service: ServeRepairService,
@@ -266,6 +273,7 @@ pub struct Validator {
     tpu: Tpu,
     tvu: Tvu,
     ip_echo_server: Option<solana_net_utils::IpEchoServer>,
+    accountsdb_plugin_service: Option<AccountsDbPluginService>,
 }
 
 // in the distant future, get rid of ::new()/exit() and use Result properly...
@@ -302,6 +310,27 @@ impl Validator {
         warn!("identity: {}", id);
         warn!("vote account: {}", vote_account);
 
+        let mut bank_notification_senders = Vec::new();
+
+        let accountsdb_plugin_service =
+            if let Some(accountsdb_plugin_config_files) = &config.accountsdb_plugin_config_files {
+                let (confirmed_bank_sender, confirmed_bank_receiver) = unbounded();
+                bank_notification_senders.push(confirmed_bank_sender);
+                let result = AccountsDbPluginService::new(
+                    confirmed_bank_receiver,
+                    accountsdb_plugin_config_files,
+                );
+                match result {
+                    Ok(accountsdb_plugin_service) => Some(accountsdb_plugin_service),
+                    Err(err) => {
+                        error!("Failed to load the AccountsDb plugin: {:?}", err);
+                        abort();
+                    }
+                }
+            } else {
+                None
+            };
+
         if config.voting_disabled {
             warn!("voting disabled");
             authorized_voter_keypairs.write().unwrap().clear();
@@ -395,10 +424,19 @@ impl Validator {
             config.enforce_ulimit_nofile,
             &start_progress,
             config.no_poh_speed_test,
+            accountsdb_plugin_service
+                .as_ref()
+                .map(|plugin_service| plugin_service.get_accounts_update_notifier()),
         );
 
         *start_progress.write().unwrap() = ValidatorStartProgress::StartingServices;
 
+        verify_udp_stats_access().unwrap_or_else(|err| {
+            error!("Failed to access UDP stats: {}", err);
+            abort();
+        });
+        let system_monitor_service = Some(SystemMonitorService::new(Arc::clone(&exit)));
+
         let leader_schedule_cache = Arc::new(leader_schedule_cache);
         let bank = bank_forks.working_bank();
         if let Some(ref shrink_paths) = config.account_shrink_paths {
@@ -532,6 +570,11 @@ impl Validator {
                 ));
             }
             let (bank_notification_sender, bank_notification_receiver) = unbounded();
+            let confirmed_bank_subscribers = if !bank_notification_senders.is_empty() {
+                Some(Arc::new(RwLock::new(bank_notification_senders)))
+            } else {
+                None
+            };
             (
                 Some(JsonRpcService::new(
                     rpc_addr,
@@ -548,8 +591,7 @@ impl Validator {
                     config.trusted_validators.clone(),
                     rpc_override_health_check.clone(),
                     optimistically_confirmed_bank.clone(),
-                    config.send_transaction_retry_ms,
-                    config.send_transaction_leader_forward_count,
+                    config.send_transaction_service_config.clone(),
                     max_slots.clone(),
                     leader_schedule_cache.clone(),
                     max_complete_transaction_status_slot,
@@ -576,6 +618,7 @@ impl Validator {
                     bank_forks.clone(),
                     optimistically_confirmed_bank,
                     rpc_subscriptions.clone(),
+                    confirmed_bank_subscribers,
                 )),
                 Some(bank_notification_sender),
             )
@@ -681,6 +724,10 @@ impl Validator {
             bank_forks.read().unwrap().root_bank().deref(),
         ));
 
+        let mut cost_model = CostModel::default();
+        cost_model.initialize_cost_table(&blockstore.read_program_costs().unwrap());
+        let cost_model = Arc::new(RwLock::new(cost_model));
+
         let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
         let (verified_vote_sender, verified_vote_receiver) = unbounded();
         let (gossip_verified_vote_hash_sender, gossip_verified_vote_hash_receiver) = unbounded();
@@ -758,6 +805,7 @@ impl Validator {
                 disable_epoch_boundary_optimization: config.disable_epoch_boundary_optimization,
             },
             &max_slots,
+            &cost_model,
         );
 
         let tpu = Tpu::new(
@@ -784,6 +832,7 @@ impl Validator {
             bank_notification_sender,
             config.tpu_coalesce_ms,
             cluster_confirmed_slot_sender,
+            &cost_model,
         );
 
         datapoint_info!("validator-new", ("id", id.to_string(), String));
@@ -798,6 +847,7 @@ impl Validator {
             transaction_status_service,
             rewards_recorder_service,
             cache_block_meta_service,
+            system_monitor_service,
             sample_performance_service,
             snapshot_packager_service,
             completed_data_sets_service,
@@ -807,6 +857,7 @@ impl Validator {
             poh_recorder,
             ip_echo_server,
             validator_exit: config.validator_exit.clone(),
+            accountsdb_plugin_service,
         }
     }
 
@@ -887,6 +938,12 @@ impl Validator {
                 .expect("cache_block_meta_service");
         }
 
+        if let Some(system_monitor_service) = self.system_monitor_service {
+            system_monitor_service
+                .join()
+                .expect("system_monitor_service");
+        }
+
         if let Some(sample_performance_service) = self.sample_performance_service {
             sample_performance_service
                 .join()
@@ -909,6 +966,12 @@ impl Validator {
         if let Some(ip_echo_server) = self.ip_echo_server {
             ip_echo_server.shutdown_background();
         }
+
+        if let Some(accountsdb_plugin_service) = self.accountsdb_plugin_service {
+            accountsdb_plugin_service
+                .join()
+                .expect("accountsdb_plugin_service");
+        }
     }
 }
 
@@ -1042,6 +1105,7 @@ fn post_process_restored_tower(
 }
 
 #[allow(clippy::type_complexity)]
+#[allow(clippy::too_many_arguments)]
 fn new_banks_from_ledger(
     validator_identity: &Pubkey,
     vote_account: &Pubkey,
@@ -1052,6 +1116,7 @@ fn new_banks_from_ledger(
     enforce_ulimit_nofile: bool,
     start_progress: &Arc<RwLock<ValidatorStartProgress>>,
     no_poh_speed_test: bool,
+    accounts_update_notifier: Option<AccountsUpdateNotifier>,
 ) -> (
     GenesisConfig,
     BankForks,
@@ -1168,6 +1233,7 @@ fn new_banks_from_ledger(
         transaction_history_services
             .cache_block_meta_sender
             .as_ref(),
+        accounts_update_notifier,
     )
     .unwrap_or_else(|err| {
         error!("Failed to load ledger: {:?}", err);
@@ -1448,76 +1514,6 @@ fn wait_for_supermajority(
     Ok(true)
 }
 
-fn is_rosetta_emulated() -> bool {
-    #[cfg(target_os = "macos")]
-    {
-        use std::str::FromStr;
-        std::process::Command::new("sysctl")
-            .args(&["-in", "sysctl.proc_translated"])
-            .output()
-            .map_err(|_| ())
-            .and_then(|output| String::from_utf8(output.stdout).map_err(|_| ()))
-            .and_then(|stdout| u8::from_str(stdout.trim()).map_err(|_| ()))
-            .map(|enabled| enabled == 1)
-            .unwrap_or(false)
-    }
-    #[cfg(not(target_os = "macos"))]
-    {
-        false
-    }
-}
-
-pub fn report_target_features() {
-    warn!(
-        "CUDA is {}abled",
-        if solana_perf::perf_libs::api().is_some() {
-            "en"
-        } else {
-            "dis"
-        }
-    );
-
-    if !is_rosetta_emulated() {
-        unsafe { check_avx() };
-        unsafe { check_avx2() };
-    }
-}
-
-// Validator binaries built on a machine with AVX support will generate invalid opcodes
-// when run on machines without AVX causing a non-obvious process abort.  Instead detect
-// the mismatch and error cleanly.
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-#[target_feature(enable = "avx")]
-unsafe fn check_avx() {
-    if is_x86_feature_detected!("avx") {
-        info!("AVX detected");
-    } else {
-        error!(
-            "Incompatible CPU detected: missing AVX support. Please build from source on the target"
-        );
-        abort();
-    }
-}
-
-#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
-unsafe fn check_avx() {}
-
-#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-#[target_feature(enable = "avx2")]
-unsafe fn check_avx2() {
-    if is_x86_feature_detected!("avx2") {
-        info!("AVX2 detected");
-    } else {
-        error!(
-            "Incompatible CPU detected: missing AVX2 support. Please build from source on the target"
-        );
-        abort();
-    }
-}
-
-#[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))]
-unsafe fn check_avx2() {}
-
 // Get the activated stake percentage (based on the provided bank) that is visible in gossip
 fn get_stake_percent_in_gossip(bank: &Bank, cluster_info: &ClusterInfo, log: bool) -> u64 {
     let mut online_stake = 0;

core/src/window_service.rs

@@ -50,6 +50,12 @@ struct WindowServiceMetrics {
     num_shreds_received: u64,
     shred_receiver_elapsed_us: u64,
     prune_shreds_elapsed_us: u64,
+    num_shreds_pruned_invalid_repair: usize,
+    num_errors: u64,
+    num_errors_blockstore: u64,
+    num_errors_cross_beam_recv_timeout: u64,
+    num_errors_other: u64,
+    num_errors_try_crossbeam_send: u64,
 }
 
 impl WindowServiceMetrics {
@@ -68,8 +74,39 @@ impl WindowServiceMetrics {
                 self.prune_shreds_elapsed_us as i64,
                 i64
             ),
+            (
+                "num_shreds_pruned_invalid_repair",
+                self.num_shreds_pruned_invalid_repair,
+                i64
+            ),
+            ("num_errors", self.num_errors, i64),
+            ("num_errors_blockstore", self.num_errors_blockstore, i64),
+            ("num_errors_other", self.num_errors_other, i64),
+            (
+                "num_errors_try_crossbeam_send",
+                self.num_errors_try_crossbeam_send,
+                i64
+            ),
+            (
+                "num_errors_cross_beam_recv_timeout",
+                self.num_errors_cross_beam_recv_timeout,
+                i64
+            ),
         );
     }
+
+    fn record_error(&mut self, err: &Error) {
+        self.num_errors += 1;
+        match err {
+            Error::TryCrossbeamSend => self.num_errors_try_crossbeam_send += 1,
+            Error::CrossbeamRecvTimeout(_) => self.num_errors_cross_beam_recv_timeout += 1,
+            Error::Blockstore(err) => {
+                self.num_errors_blockstore += 1;
+                error!("blockstore error: {}", err);
+            }
+            _ => self.num_errors_other += 1,
+        }
+    }
 }
 
 #[derive(Default)]
@@ -263,6 +300,7 @@ fn run_insert<F>(
 where
     F: Fn(Shred),
 {
+    ws_metrics.run_insert_count += 1;
     let mut shred_receiver_elapsed = Measure::start("shred_receiver_elapsed");
     let timer = Duration::from_millis(200);
     let (mut shreds, mut repair_infos) = shred_receiver.recv_timeout(timer)?;
@@ -271,15 +309,19 @@ where
         repair_infos.extend(more_repair_infos);
     }
     shred_receiver_elapsed.stop();
+    ws_metrics.shred_receiver_elapsed_us += shred_receiver_elapsed.as_us();
     ws_metrics.num_shreds_received += shreds.len() as u64;
 
     let mut prune_shreds_elapsed = Measure::start("prune_shreds_elapsed");
+    let num_shreds = shreds.len();
     prune_shreds_invalid_repair(&mut shreds, &mut repair_infos, outstanding_requests);
+    ws_metrics.num_shreds_pruned_invalid_repair = num_shreds - shreds.len();
     let repairs: Vec<_> = repair_infos
         .iter()
         .map(|repair_info| repair_info.is_some())
         .collect();
     prune_shreds_elapsed.stop();
+    ws_metrics.prune_shreds_elapsed_us += prune_shreds_elapsed.as_us();
 
     let (completed_data_sets, inserted_indices) = blockstore.insert_shreds_handle_duplicate(
         shreds,
@@ -297,11 +339,6 @@ where
     }
 
     completed_data_sets_sender.try_send(completed_data_sets)?;
-
-    ws_metrics.run_insert_count += 1;
-    ws_metrics.shred_receiver_elapsed_us += shred_receiver_elapsed.as_us();
-    ws_metrics.prune_shreds_elapsed_us += prune_shreds_elapsed.as_us();
-
     Ok(())
 }
 
@@ -561,6 +598,7 @@ impl WindowService {
                         &retransmit_sender,
                         &outstanding_requests,
                     ) {
+                        ws_metrics.record_error(&e);
                         if Self::should_exit_on_error(e, &mut handle_timeout, &handle_error) {
                             break;
                         }

core/tests/snapshots.rs

@@ -109,6 +109,7 @@ mod tests {
                 false,
                 accounts_db::AccountShrinkThreshold::default(),
                 false,
+                None,
             );
             bank0.freeze();
             let mut bank_forks = BankForks::new(bank0);
@@ -172,6 +173,7 @@ mod tests {
             accounts_db::AccountShrinkThreshold::default(),
             check_hash_calculation,
             false,
+            None,
         )
         .unwrap();
 

crate-features/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-crate-features"
-version = "1.7.15"
+version = "1.8.1"
 description = "Solana Crate Features"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

docs/sidebars.js

@@ -41,7 +41,6 @@ module.exports = {
       "cli/choose-a-cluster",
       "cli/transfer-tokens",
       "cli/delegate-stake",
-      "cli/manage-stake-accounts",
       "cli/deploy-a-program",
       "offline-signing",
       "offline-signing/durable-nonce",
@@ -65,6 +64,7 @@ module.exports = {
         items: [
           "developing/clients/jsonrpc-api",
           "developing/clients/javascript-api",
+          "developing/clients/javascript-reference",
           "developing/clients/rust-api",
         ],
       },

docs/src/cli/manage-stake-accounts.md

@@ -1,78 +0,0 @@
----
-title: Manage Stake Accounts
----
-
-If you want to delegate stake to many different validators, you will need
-to create a separate stake account for each. If you follow the convention
-of creating the first stake account at seed "0", the second at "1", the
-third at "2", and so on, then the `solana-stake-accounts` tool will allow
-you to operate on all accounts with single invocations. You can use it to
-sum up the balances of all accounts, move accounts to a new wallet, or set
-new authorities.
-
-## Usage
-
-### Create a stake account
-
-Create and fund a derived stake account at the stake authority public key:
-
-```bash
-solana-stake-accounts new <FUNDING_KEYPAIR> <BASE_KEYPAIR> <AMOUNT> \
-    --stake-authority <PUBKEY> --withdraw-authority <PUBKEY> \
-    --fee-payer <KEYPAIR>
-```
-
-### Count accounts
-
-Count the number of derived accounts:
-
-```bash
-solana-stake-accounts count <BASE_PUBKEY>
-```
-
-### Get stake account balances
-
-Sum the balance of derived stake accounts:
-
-```bash
-solana-stake-accounts balance <BASE_PUBKEY> --num-accounts <NUMBER>
-```
-
-### Get stake account addresses
-
-List the address of each stake account derived from the given public key:
-
-```bash
-solana-stake-accounts addresses <BASE_PUBKEY> --num-accounts <NUMBER>
-```
-
-### Set new authorities
-
-Set new authorities on each derived stake account:
-
-```bash
-solana-stake-accounts authorize <BASE_PUBKEY> \
-    --stake-authority <KEYPAIR> --withdraw-authority <KEYPAIR> \
-    --new-stake-authority <PUBKEY> --new-withdraw-authority <PUBKEY> \
-    --num-accounts <NUMBER> --fee-payer <KEYPAIR>
-```
-
-### Relocate stake accounts
-
-Relocate stake accounts:
-
-```bash
-solana-stake-accounts rebase <BASE_PUBKEY> <NEW_BASE_KEYPAIR> \
-    --stake-authority <KEYPAIR> --num-accounts <NUMBER> \
-    --fee-payer <KEYPAIR>
-```
-
-To atomically rebase and authorize each stake account, use the 'move'
-command:
-
-```bash
-solana-stake-accounts move <BASE_PUBKEY> <NEW_BASE_KEYPAIR> \
-    --stake-authority <KEYPAIR> --withdraw-authority <KEYPAIR> \
-    --new-stake-authority <PUBKEY> --new-withdraw-authority <PUBKEY> \
-    --num-accounts <NUMBER> --fee-payer <KEYPAIR>
-```

docs/src/developing/backwards-compatibility.md

@@ -76,6 +76,7 @@ Major releases:
 - [`solana-program`](https://docs.rs/solana-program/) - Rust SDK for writing programs
 - [`solana-client`](https://docs.rs/solana-client/) - Rust client for connecting to RPC API
 - [`solana-cli-config`](https://docs.rs/solana-cli-config/) - Rust client for managing Solana CLI config files
+- [`solana-accountsdb-plugin-interface`](https://docs.rs/solana-accountsdb-plugin-interface/) - Rust interface for developing Solana AccountsDb plugins.
 
 Patch releases:
 

docs/src/developing/clients/javascript-api.md

@@ -2,4 +2,331 @@
 title: Web3 JavaScript API
 ---
 
-See [solana-web3](https://solana-labs.github.io/solana-web3.js/).
+## What is Solana-Web3.js?
+
+The Solana-Web3.js library aims to provide complete coverage of Solana. The library was built on top of the [Solana JSON RPC API](https://docs.solana.com/developing/clients/jsonrpc-api).
+
+## Common Terminology
+
+| Term | Definition |
+|-------------|------------------------|
+| Program     | Stateless executable code written to interpret instructions. Programs are capable of performing actions based on the instructions provided. |
+| Instruction | The smallest unit of a program that a client can include in a transaction. Within its processing code, an instruction may contain one or more cross-program invocations. |
+| Transaction | One or more instructions signed by the client using one or more Keypairs and executed atomically with only two possible outcomes: success or failure. |
+
+For the full list of terms, see [Solana terminology](https://docs.solana.com/terminology#cross-program-invocation)
+
+## Getting Started
+
+### Installation
+
+#### yarn
+
+```bash
+$ yarn add @solana/web3.js
+```
+
+#### npm
+
+```bash
+$ npm install --save @solana/web3.js
+```
+
+#### Bundle
+
+```html
+<!-- Development (un-minified) -->
+<script src="https://unpkg.com/@solana/web3.js@latest/lib/index.iife.js"></script>
+
+<!-- Production (minified) -->
+<script src="https://unpkg.com/@solana/web3.js@latest/lib/index.iife.min.js"></script>
+```
+
+### Usage
+
+#### Javascript
+
+```javascript
+const solanaWeb3 = require('@solana/web3.js');
+console.log(solanaWeb3);
+```
+
+
+#### ES6
+
+```javascript
+import * as solanaWeb3 from '@solana/web3.js';
+console.log(solanaWeb3);
+```
+
+
+#### Browser Bundle
+
+```javascript
+// solanaWeb3 is provided in the global namespace by the bundle script
+console.log(solanaWeb3);
+```
+
+## Quickstart
+
+### Connecting to a Wallet
+
+To allow users to use your dApp or application on Solana, they will need to get access to their Keypair. A Keypair is a private key with a matching public key, used to sign transactions.
+
+There are two ways to obtain a Keypair:
+1. Generate a new Keypair
+2. Obtain a Keypair using the secret key
+
+You can obtain a new Keypair with the following:
+
+```javascript
+const {Keypair} = require("@solana/web3.js");
+
+let keypair = Keypair.generate();
+```
+
+This will generate a brand new Keypair for a user to fund and use within your application.
+
+You can allow entry of the secretKey using a textbox, and obtain the Keypair with `Keypair.fromSecretKey(secretKey)`.
+
+```javascript
+const {Keypair} = require("@solana/web3.js");
+
+let secretKey = Uint8Array.from([
+  202, 171, 192, 129, 150, 189, 204, 241, 142,  71, 205,
+  2,  81,  97,   2, 176,  48,  81,  45,   1,  96, 138,
+  220, 132, 231, 131, 120,  77,  66,  40,  97, 172,  91,
+  245,  84, 221, 157, 190,   9, 145, 176, 130,  25,  43,
+  72, 107, 190, 229,  75,  88, 191, 136,   7, 167, 109,
+  91, 170, 164, 186,  15, 142,  36,  12,  23
+]);
+
+let keypair = Keypair.fromSecretKey(secretKey);
+```
+
+Many wallets today allow users to bring their Keypair using a variety of extensions or web wallets. The general recommendation is to use wallets, not Keypairs, to sign transactions. The wallet creates a layer of separation between the dApp and the Keypair, ensuring that the dApp never has access to the secret key. You can find ways to connect to external wallets with the [wallet-adapter](https://github.com/solana-labs/wallet-adapter) library.
+
+### Creating and Sending Transactions
+
+To interact with programs on Solana, you create, sign, and send transactions to the network. Transactions are collections of instructions with signatures. The order that instructions exist in a transaction determines the order they are executed.
+
+A transaction in Solana-Web3.js is created using the [`Transaction`](javascript-api.md#Transaction) object and adding desired messages, addresses, or instructions.
+
+Take the example of a transfer transaction:
+
+```javascript
+const {Keypair, Transaction, SystemProgram, LAMPORTS_PER_SOL} = require("@solana/web3.js");
+
+let fromKeypair = Keypair.generate();
+let toKeypair = Keypair.generate();
+let transaction = new Transaction();
+
+transaction.add(
+  SystemProgram.transfer({
+    fromPubkey: fromKeypair.publicKey,
+    toPubkey: toKeypair.publicKey,
+    lamports: LAMPORTS_PER_SOL
+  })
+);
+```
+
+The above code achieves creating a transaction ready to be signed and broadcasted to the network. The `SystemProgram.transfer` instruction was added to the transaction, containing the amount of lamports to send, and the `to` and `from` public keys.
+
+All that is left is to sign the transaction with keypair and send it over the network. You can accomplish sending a transaction by using `sendAndConfirmTransaction` if you wish to alert the user or do something after a transaction is finished, or use `sendTransaction` if you don't need to wait for the transaction to be confirmed.
+
+```javascript
+const {sendAndConfirmTransaction, clusterApiUrl} = require("@solana/web3.js");
+
+let keypair = Keypair.generate();
+let connection = new Connection(clusterApiUrl('testnet'));
+
+sendAndConfirmTransaction(
+  connection,
+  transaction,
+  [keypair]
+);
+```
+
+The above code takes in a `TransactionInstruction` using `SystemProgram`, creates a `Transaction`, and sends it over the network. You use `Connection` in order to define with Solana network you are connecting to, namely `mainnet-beta`, `testnet`, or `devnet`.
+
+### Interacting with Custom Programs
+
+The previous section visits sending basic transactions. In Solana everything you do interacts with different programs, including the previous section's transfer transaction. At the time of writing programs on Solana are either written in Rust or C.
+
+Let's look at the `SystemProgram`. The method signature for allocating space in your account on Solana in Rust looks like this:
+
+```rust
+pub fn allocate(
+    pubkey: &Pubkey,
+    space: u64
+) -> Instruction
+```
+
+In Solana when you want to interact with a program you must first know all the accounts you will be interacting with.
+
+You must always provide every account that the program will be interacting within the instruction. Not only that, but you must provide whether or not the account is `isSigner` or `isWritable`.
+
+In the `allocate` method above, a single account `pubkey` is required, as well as an amount of `space` for allocation. We know that the `allocate` method writes to the account by allocating space within it, making the `pubkey` required to be `isWritable`. `isSigner` is required when you are designating the account that is running the instruction. In this case, the signer is the account calling to allocate space within itself.
+
+Let's look at how to call this instruction using solana-web3.js:
+
+```javascript
+let keypair = web3.Keypair.generate();
+let payer = web3.Keypair.generate();
+let connection = new web3.Connection(web3.clusterApiUrl('testnet'));
+
+let airdropSignature = await connection.requestAirdrop(
+  payer.publicKey,
+  web3.LAMPORTS_PER_SOL,
+);
+
+await connection.confirmTransaction(airdropSignature);
+```
+
+First, we set up the account Keypair and connection so that we have an account to make allocate on the testnet. We also create a payer Keypair and airdrop some sol so we can pay for the allocate transaction.
+
+```javascript
+let allocateTransaction = new web3.Transaction({
+    feePayer: payer.publicKey
+  });
+let keys = [{pubkey: keypair.publicKey, isSigner: true, isWritable: true}];
+let params = { space: 100 };
+```
+
+We create the transaction `allocateTransaction`, keys, and params objects. `feePayer` is an optional field when creating a transaction that specifies who is paying for the transaction, defaulting to the pubkey of the first signer in the transaction. `keys` represents all accounts that the program's `allocate` function will interact with. Since the `allocate` function also required space, we created `params` to be used later when invoking the `allocate` function.
+
+```javascript
+let allocateStruct = {
+  index: 8,
+  layout: struct([
+    u32('instruction'),
+    ns64('space'),
+  ])
+};
+```
+
+The above is created using using u32 and ns64 from `@solana/buffer-layout` to facilitate the payload creation. The `allocate` function takes in the parameter `space`. To interact with the function we must provide the data as a Buffer format. The `buffer-layout` library helps with allocating the buffer and encoding it correctly for Rust programs on Solana to interpret.
+
+Let's break down this struct.
+
+```javascript
+{
+  index: 8, /* <-- */
+  layout: struct([
+    u32('instruction'),
+    ns64('space'),
+  ])
+}
+```
+
+`index` is set to 8 because the function `allocate` is in the 8th position in the instruction enum for `SystemProgram`.
+
+```rust
+/* https://github.com/solana-labs/solana/blob/21bc43ed58c63c827ba4db30426965ef3e807180/sdk/program/src/system_instruction.rs#L142-L305 */
+pub enum SystemInstruction {
+    /** 0 **/CreateAccount {/**/},
+    /** 1 **/Assign {/**/},
+    /** 2 **/Transfer {/**/},
+    /** 3 **/CreateAccountWithSeed {/**/},
+    /** 4 **/AdvanceNonceAccount,
+    /** 5 **/WithdrawNonceAccount(u64),
+    /** 6 **/InitializeNonceAccount(Pubkey),
+    /** 7 **/AuthorizeNonceAccount(Pubkey),
+    /** 8 **/Allocate {/**/},
+    /** 9 **/AllocateWithSeed {/**/},
+    /** 10 **/AssignWithSeed {/**/},
+    /** 11 **/TransferWithSeed {/**/},
+}
+```
+
+Next up is `u32('instruction')`.
+
+```javascript
+{
+  index: 8,
+  layout: struct([
+    u32('instruction'), /* <-- */
+    ns64('space'),
+  ])
+}
+```
+
+The `layout` in the allocate struct must always have `u32('instruction')` first when you are using it to call an instruction.
+
+```javascript
+{
+  index: 8,
+  layout: struct([
+    u32('instruction'),
+    ns64('space'), /* <-- */
+  ])
+}
+```
+
+`ns64('space')` is the argument for the `allocate` function. You can see in the original `allocate` function in Rust that space was of the type `u64`. `u64` is an unsigned 64bit integer. Javascript by default only provides up to 53bit integers. `ns64` comes from `@solana/buffer-layout` to help with type conversions between Rust and Javascript. You can find more type conversions between Rust and Javascript at [solana-labs/buffer-layout](https://github.com/solana-labs/buffer-layout).
+
+```javascript
+let data = Buffer.alloc(allocateStruct.layout.span);
+let layoutFields = Object.assign({instruction: allocateStruct.index}, params);
+allocateStruct.layout.encode(layoutFields, data);
+```
+
+Using the previously created bufferLayout, we can allocate a data buffer. We then assign our params `{ space: 100 }` so that it maps correctly to the layout, and encode it to the data buffer. Now the data is ready to be sent to the program.
+
+```javascript
+allocateTransaction.add(new web3.TransactionInstruction({
+  keys,
+  programId: web3.SystemProgram.programId,
+  data,
+}));
+
+await web3.sendAndConfirmTransaction(connection, allocateTransaction, [payer, keypair]);
+```
+
+Finally, we add the transaction instruction with all the account keys, payer, data, and programId and broadcast the transaction to the network.
+
+The full code can be found below.
+
+```javascript
+const {struct, u32, ns64} = require("@solana/buffer-layout");
+const {Buffer} = require('buffer');
+const web3 = require("@solana/web3.js");
+
+let keypair = web3.Keypair.generate();
+let payer = web3.Keypair.generate();
+
+let connection = new web3.Connection(web3.clusterApiUrl('testnet'));
+
+let airdropSignature = await connection.requestAirdrop(
+  payer.publicKey,
+  web3.LAMPORTS_PER_SOL,
+);
+
+await connection.confirmTransaction(airdropSignature);
+
+let allocateTransaction = new web3.Transaction({
+  feePayer: payer.publicKey
+});
+let keys = [{pubkey: keypair.publicKey, isSigner: true, isWritable: true}];
+let params = { space: 100 };
+
+let allocateStruct = {
+  index: 8,
+  layout: struct([
+    u32('instruction'),
+    ns64('space'),
+  ])
+};
+
+let data = Buffer.alloc(allocateStruct.layout.span);
+let layoutFields = Object.assign({instruction: allocateStruct.index}, params);
+allocateStruct.layout.encode(layoutFields, data);
+
+allocateTransaction.add(new web3.TransactionInstruction({
+  keys,
+  programId: web3.SystemProgram.programId,
+  data,
+}));
+
+await web3.sendAndConfirmTransaction(connection, allocateTransaction, [payer, keypair]);
+```

docs/src/developing/clients/javascript-reference.md

@@ -0,0 +1,729 @@
+---
+title: Web3 API Reference
+---
+
+## Web3 API Reference Guide
+
+## General
+
+### Connection
+
+[Source Documentation](https://solana-labs.github.io/solana-web3.js/classes/Connection.html)
+
+Connection is used to interact with the [Solana JSON RPC](https://docs.solana.com/developing/clients/jsonrpc-api). You can use Connection to confirm transactions, get account info, and more.
+
+You create a connection by defining the JSON RPC cluster endpoint and the desired commitment. Once this is complete, you can use this connection object to interact with any of the Solana JSON RPC API.
+
+#### Example Usage
+
+```javascript
+const web3 = require("@solana/web3.js");
+
+let connection = new web3.Connection(web3.clusterApiUrl('devnet'), 'confirmed');
+
+let slot = await connection.getSlot();
+console.log(slot);
+// 93186439
+
+let blockTime = await connection.getBlockTime(slot);
+console.log(blockTime);
+// 1630747045
+
+let block = await connection.getBlock(slot);
+console.log(block);
+
+/*
+{
+    blockHeight: null,
+    blockTime: 1630747045,
+    blockhash: 'AsFv1aV5DGip9YJHHqVjrGg6EKk55xuyxn2HeiN9xQyn',
+    parentSlot: 93186438,
+    previousBlockhash: '11111111111111111111111111111111',
+    rewards: [],
+    transactions: []
+}
+*/
+
+let slotLeader = await connection.getSlotLeader();
+console.log(slotLeader);
+//49AqLYbpJYc2DrzGUAH1fhWJy62yxBxpLEkfJwjKy2jr
+```
+
+The above example shows only a few of the methods on Connection. Please see the [source generated docs](https://solana-labs.github.io/solana-web3.js/classes/Connection.html) for the full list.
+
+### Transaction
+
+[SourceDocumentation](https://solana-labs.github.io/solana-web3.js/classes/Transaction.html)
+
+A transaction is used to interact with programs on the Solana blockchain. These transactions are constructed with TransactionInstructions, containing all the accounts possible to interact with, as well as any needed data or program addresses. Each TransactionInstruction consists of keys, data, and a programId. You can do multiple instructions in a single transaction, interacting with multiple programs at once.
+
+#### Example Usage
+
+```javascript
+const web3 = require('@solana/web3.js');
+const nacl = require('tweetnacl');
+
+// Airdrop SOL for paying transactions
+let payer = web3.Keypair.generate();
+let connection = new web3.Connection(web3.clusterApiUrl('devnet'), 'confirmed');
+
+let airdropSignature = await connection.requestAirdrop(
+    payer.publicKey,
+    web3.LAMPORTS_PER_SOL,
+);
+
+await connection.confirmTransaction(airdropSignature);
+
+let toAccount = web3.Keypair.generate();
+
+// Create Simple Transaction
+let transaction = new web3.Transaction();
+
+// Add an instruction to execute
+transaction.add(web3.SystemProgram.transfer({
+    fromPubkey: payer.publicKey,
+    toPubkey: toAccount.publicKey,
+    lamports: 1000,
+}));
+
+// Send and confirm transaction
+// Note: feePayer is by default the first signer, or payer, if the parameter is not set
+await web3.sendAndConfirmTransaction(connection, transaction, [payer])
+
+// Alternatively, manually construct the transaction
+let recentBlockhash = await connection.getRecentBlockhash();
+let manualTransaction = new web3.Transaction({
+    recentBlockhash: recentBlockhash.blockhash,
+    feePayer: payer.publicKey
+});
+manualTransaction.add(web3.SystemProgram.transfer({
+    fromPubkey: payer.publicKey,
+    toPubkey: toAccount.publicKey,
+    lamports: 1000,
+}));
+
+let transactionBuffer = manualTransaction.serializeMessage();
+let signature = nacl.sign.detached(transactionBuffer, payer.secretKey);
+
+manualTransaction.addSignature(payer.publicKey, signature);
+
+let isVerifiedSignature = manualTransaction.verifySignatures();
+console.log(`The signatures were verifed: ${isVerifiedSignature}`)
+
+// The signatures were verified: true
+
+let rawTransaction = manualTransaction.serialize();
+
+await web3.sendAndConfirmRawTransaction(connection, rawTransaction);
+```
+
+### Keypair
+
+[Source Documentation](https://solana-labs.github.io/solana-web3.js/classes/Keypair.html)
+
+The keypair is used to create an account with a public key and secret key within Solana. You can either generate, generate from a seed, or create from a secret key.
+
+#### Example Usage
+
+```javascript
+const {Keypair} = require("@solana/web3.js")
+
+let account = Keypair.generate();
+
+console.log(account.publicKey.toBase58());
+console.log(account.secretKey);
+
+// 2DVaHtcdTf7cm18Zm9VV8rKK4oSnjmTkKE6MiXe18Qsb
+// Uint8Array(64) [
+//   152,  43, 116, 211, 207,  41, 220,  33, 193, 168, 118,
+//    24, 176,  83, 206, 132,  47, 194,   2, 203, 186, 131,
+//   197, 228, 156, 170, 154,  41,  56,  76, 159, 124,  18,
+//    14, 247,  32, 210,  51, 102,  41,  43,  21,  12, 170,
+//   166, 210, 195, 188,  60, 220, 210,  96, 136, 158,   6,
+//   205, 189, 165, 112,  32, 200, 116, 164, 234
+// ]
+
+
+let seed = Uint8Array.from([70,60,102,100,70,60,102,100,70,60,102,100,70,60,102,100,70,60,102,100,70,60,102,100,70,60,102,100,70,60,102,100]);
+let accountFromSeed = Keypair.fromSeed(seed);
+
+console.log(accountFromSeed.publicKey.toBase58());
+console.log(accountFromSeed.secretKey);
+
+// 3LDverZtSC9Duw2wyGC1C38atMG49toPNW9jtGJiw9Ar
+// Uint8Array(64) [
+//    70,  60, 102, 100,  70,  60, 102, 100,  70,  60, 102,
+//   100,  70,  60, 102, 100,  70,  60, 102, 100,  70,  60,
+//   102, 100,  70,  60, 102, 100,  70,  60, 102, 100,  34,
+//   164,   6,  12,   9, 193, 196,  30, 148, 122, 175,  11,
+//    28, 243, 209,  82, 240, 184,  30,  31,  56, 223, 236,
+//   227,  60,  72, 215,  47, 208, 209, 162,  59
+// ]
+
+
+let accountFromSecret = Keypair.fromSecretKey(account.secretKey);
+
+console.log(accountFromSecret.publicKey.toBase58());
+console.log(accountFromSecret.secretKey);
+
+// 2DVaHtcdTf7cm18Zm9VV8rKK4oSnjmTkKE6MiXe18Qsb
+// Uint8Array(64) [
+//   152,  43, 116, 211, 207,  41, 220,  33, 193, 168, 118,
+//    24, 176,  83, 206, 132,  47, 194,   2, 203, 186, 131,
+//   197, 228, 156, 170, 154,  41,  56,  76, 159, 124,  18,
+//    14, 247,  32, 210,  51, 102,  41,  43,  21,  12, 170,
+//   166, 210, 195, 188,  60, 220, 210,  96, 136, 158,   6,
+//   205, 189, 165, 112,  32, 200, 116, 164, 234
+// ]
+```
+
+Using `generate` generates a random Keypair for use as an account on Solana. Using `fromSeed`, you can generate a Keypair using a deterministic constructor. `fromSecret` creates a Keypair from a secret Uint8array. You can see that the publicKey for the `generate` Keypair and `fromSecret` Keypair are the same because the secret from the `generate` Keypair is used in `fromSecret`.
+
+**Warning**: Do not use `fromSeed` unless you are creating a seed with high entropy. Do not share your seed. Treat the seed like you would a private key.
+
+### PublicKey
+
+[Source Documentation](https://solana-labs.github.io/solana-web3.js/classes/PublicKey.html)
+
+PublicKey is used throughout `@solana/web3.js` in transactions, keypairs, and programs. You require publickey when listing each account in a transaction and as a general identifier on Solana.
+
+A PublicKey can be created with a base58 encoded string, buffer, Uint8Array, number, and an array of numbers.
+
+#### Example Usage
+
+```javascript
+const {Buffer} = require('buffer');
+const web3 = require('@solana/web3.js');
+const crypto = require('crypto');
+
+// Create a PublicKey with a base58 encoded string
+let base58publicKey = new web3.PublicKey('5xot9PVkphiX2adznghwrAuxGs2zeWisNSxMW6hU6Hkj');
+console.log(base58publicKey.toBase58());
+
+// 5xot9PVkphiX2adznghwrAuxGs2zeWisNSxMW6hU6Hkj
+
+// Create a Program Address
+let highEntropyBuffer = crypto.randomBytes(31);
+let programAddressFromKey = await web3.PublicKey.createProgramAddress([highEntropyBuffer.slice(0, 31)], base58publicKey);
+console.log(`Generated Program Address: ${programAddressFromKey.toBase58()}`);
+
+// Generated Program Address: 3thxPEEz4EDWHNxo1LpEpsAxZryPAHyvNVXJEJWgBgwJ
+
+// Find Program address given a PublicKey
+let validProgramAddress = await web3.PublicKey.findProgramAddress([Buffer.from('', 'utf8')], programAddressFromKey);
+console.log(`Valid Program Address: ${validProgramAddress}`);
+
+// Valid Program Address: C14Gs3oyeXbASzwUpqSymCKpEyccfEuSe8VRar9vJQRE,253
+```
+
+### SystemProgram
+
+[SourceDocumentation](https://solana-labs.github.io/solana-web3.js/classes/SystemProgram.html)
+
+The SystemProgram grants the ability to create accounts, allocate account data, assign an account to programs, work with nonce accounts, and transfer lamports. You can use the SystemInstruction class to help with decoding and reading individual instructions
+
+#### Example Usage
+
+```javascript
+const web3 = require("@solana/web3.js");
+
+// Airdrop SOL for paying transactions
+let payer = web3.Keypair.generate();
+let connection = new web3.Connection(web3.clusterApiUrl('devnet'), 'confirmed');
+
+let airdropSignature = await connection.requestAirdrop(
+    payer.publicKey,
+    web3.LAMPORTS_PER_SOL,
+);
+
+await connection.confirmTransaction(airdropSignature);
+
+// Allocate Account Data
+let allocatedAccount = web3.Keypair.generate();
+let allocateInstruction = web3.SystemProgram.allocate({
+    accountPubkey: allocatedAccount.publicKey,
+    space: 100,
+})
+let transaction = new web3.Transaction().add(allocateInstruction);
+
+await web3.sendAndConfirmTransaction(connection, transaction, [payer, allocatedAccount])
+
+// Create Nonce Account
+let nonceAccount = web3.Keypair.generate();
+let minimumAmountForNonceAccount = await connection.getMinimumBalanceForRentExemption(
+    web3.NONCE_ACCOUNT_LENGTH,
+);
+let createNonceAccountTransaction = new web3.Transaction().add(
+web3.SystemProgram.createNonceAccount({
+    fromPubkey: payer.publicKey,
+    noncePubkey: nonceAccount.publicKey,
+    authorizedPubkey: payer.publicKey,
+    lamports: minimumAmountForNonceAccount,
+}),
+);
+
+await web3.sendAndConfirmTransaction(connection, createNonceAccountTransaction, [payer, nonceAccount])
+
+// Advance nonce - Used to create transactions as an account custodian
+let advanceNonceTransaction = new web3.Transaction().add(
+    web3.SystemProgram.nonceAdvance({
+        noncePubkey: nonceAccount.publicKey,
+        authorizedPubkey: payer.publicKey,
+    }),
+);
+
+await web3.sendAndConfirmTransaction(connection, advanceNonceTransaction, [payer])
+
+// Transfer lamports between accounts
+let toAccount = web3.Keypair.generate();
+
+let transferTransaction = new web3.Transaction().add(
+web3.SystemProgram.transfer({
+    fromPubkey: payer.publicKey,
+    toPubkey: toAccount.publicKey,
+    lamports: 1000,
+}),
+);
+await web3.sendAndConfirmTransaction(connection, transferTransaction, [payer])
+
+// Assign a new account to a program
+let programId = web3.Keypair.generate();
+let assignedAccount = web3.Keypair.generate();
+
+let assignTransaction = new web3.Transaction().add(
+web3.SystemProgram.assign({
+    accountPubkey: assignedAccount.publicKey,
+    programId: programId.publicKey,
+}),
+);
+
+await web3.sendAndConfirmTransaction(connection, assignTransaction, [payer, assignedAccount]);
+```
+
+### Secp256k1Program
+
+[Source Documentation](https://solana-labs.github.io/solana-web3.js/classes/Secp256k1Program.html)
+
+The Secp256k1Program is used to verify Secp256k1 signatures, which are used by both Bitcoin and Ethereum.
+
+#### Example Usage
+
+```javascript
+const {keccak_256} = require('js-sha3');
+const web3 = require("@solana/web3.js");
+const secp256k1 = require('secp256k1');
+
+// Create a Ethereum Address from secp256k1
+let secp256k1PrivateKey;
+do {
+    secp256k1PrivateKey = web3.Keypair.generate().secretKey.slice(0, 32);
+} while (!secp256k1.privateKeyVerify(secp256k1PrivateKey));
+
+let secp256k1PublicKey = secp256k1.publicKeyCreate(secp256k1PrivateKey, false).slice(1);
+
+let ethAddress = web3.Secp256k1Program.publicKeyToEthAddress(secp256k1PublicKey);
+console.log(`Ethereum Address: 0x${ethAddress.toString('hex')}`);
+
+// Ethereum Address: 0xadbf43eec40694eacf36e34bb5337fba6a2aa8ee
+
+// Fund a keypair to create instructions
+let fromPublicKey = web3.Keypair.generate();
+let connection = new web3.Connection(web3.clusterApiUrl('devnet'), 'confirmed');
+
+let airdropSignature = await connection.requestAirdrop(
+    fromPublicKey.publicKey,
+    web3.LAMPORTS_PER_SOL,
+);
+await connection.confirmTransaction(airdropSignature);
+
+// Sign Message with Ethereum Key
+let plaintext = Buffer.from('string address');
+let plaintextHash = Buffer.from(keccak_256.update(plaintext).digest());
+let {signature, recid: recoveryId} = secp256k1.ecdsaSign(
+    plaintextHash,
+    secp256k1PrivateKey
+);
+
+// Create transaction to verify the signature
+let transaction = new Transaction().add(
+    web3.Secp256k1Program.createInstructionWithEthAddress({
+        ethAddress: ethAddress.toString('hex'),
+        plaintext,
+        signature,
+        recoveryId,
+    }),
+);
+
+// Transaction will succeed if the message is verified to be signed by the address
+await web3.sendAndConfirmTransaction(connection, transaction, [fromPublicKey]);
+```
+
+### Message
+
+[Source Documentation](https://solana-labs.github.io/solana-web3.js/classes/Message.html)
+
+Message is used as another way to construct transactions. You can construct a message using the accounts, header, instructions, and recentBlockhash that are a part of a transaction. A [Transaction](https://solana-labs.github.io/solana-web3.js/classes/Transaction.html) is a Message plus the list of required signatures required to execute the transaction.
+
+#### Example Usage
+
+```javascript
+const {Buffer} = require("buffer");
+const bs58 = require('bs58');
+const web3 = require('@solana/web3.js');
+
+let toPublicKey = web3.Keypair.generate().publicKey;
+let fromPublicKey = web3.Keypair.generate();
+
+let connection = new web3.Connection(
+    web3.clusterApiUrl('devnet'),
+    'confirmed'
+);
+
+let airdropSignature = await connection.requestAirdrop(
+    fromPublicKey.publicKey,
+    web3.LAMPORTS_PER_SOL,
+);
+
+await connection.confirmTransaction(airdropSignature);
+
+let type = web3.SYSTEM_INSTRUCTION_LAYOUTS.Transfer;
+let data = Buffer.alloc(type.layout.span);
+let layoutFields = Object.assign({instruction: type.index});
+type.layout.encode(layoutFields, data);
+
+let recentBlockhash = await connection.getRecentBlockhash();
+
+let messageParams = {
+    accountKeys: [
+        fromPublicKey.publicKey.toString(),
+        toPublicKey.toString(),
+        web3.SystemProgram.programId.toString()
+    ],
+    header: {
+        numReadonlySignedAccounts: 0,
+        numReadonlyUnsignedAccounts: 1,
+        numRequiredSignatures: 1,
+    },
+    instructions: [
+        {
+        accounts: [0, 1],
+        data: bs58.encode(data),
+        programIdIndex: 2,
+        },
+    ],
+    recentBlockhash,
+};
+
+let message = new web3.Message(messageParams);
+
+let transaction = web3.Transaction.populate(
+    message,
+    [fromPublicKey.publicKey.toString()]
+);
+
+await web3.sendAndConfirmTransaction(connection, transaction, [fromPublicKey])
+```
+
+### Struct
+
+[SourceDocumentation](https://solana-labs.github.io/solana-web3.js/classes/Struct.html)
+
+The struct class is used to create Rust compatible structs in javascript. This class is only compatible with Borsch encoded Rust structs.
+
+#### Example Usage
+
+Struct in Rust:
+```rust
+pub struct Fee {
+    pub denominator: u64,
+    pub numerator: u64,
+}
+```
+
+Using web3:
+```javascript
+import BN from 'bn.js';
+import {Struct} from '@solana/web3.js';
+
+export class Fee extends Struct {
+  denominator: BN;
+  numerator: BN;
+}
+```
+
+### Enum
+
+[Source Documentation](https://solana-labs.github.io/solana-web3.js/classes/Enum.html)
+
+The Enum class is used to represent a Rust compatible Enum in javascript. The enum will just be a string representation if logged but can be properly encoded/decoded when used in conjunction with [Struct](https://solana-labs.github.io/solana-web3.js/classes/Struct.html). This class is only compatible with Borsch encoded Rust enumerations.
+
+#### Example Usage
+
+Rust:
+```rust
+pub enum AccountType {
+    Uninitialized,
+    StakePool,
+    ValidatorList,
+}
+```
+
+Web3:
+```javascript
+import {Enum} from '@solana/web3.js';
+
+export class AccountType extends Enum {}
+```
+
+### NonceAccount
+
+[Source Documentation](https://solana-labs.github.io/solana-web3.js/classes/NonceAccount.html)
+
+Normally a transaction is rejected if a transaction's `recentBlockhash` field is too old. To provide for certain custodial services, Nonce Accounts are used. Transactions which use a `recentBlockhash` captured on-chain by a Nonce Account do not expire as long at the Nonce Account is not advanced.
+
+You can create a nonce account by first creating a normal account, then using `SystemProgram` to make the account a Nonce Account.
+
+#### Example Usage
+
+```javascript
+const web3 = require('@solana/web3.js');
+
+// Create connection
+let connection = new web3.Connection(
+    web3.clusterApiUrl('devnet'),
+    'confirmed',
+);
+
+// Generate accounts
+let account = web3.Keypair.generate();
+let nonceAccount = web3.Keypair.generate();
+
+// Fund account
+let airdropSignature = await connection.requestAirdrop(
+    account.publicKey,
+    web3.LAMPORTS_PER_SOL,
+);
+
+await connection.confirmTransaction(airdropSignature);
+
+// Get Minimum amount for rent exemption
+let minimumAmount = await connection.getMinimumBalanceForRentExemption(
+    web3.NONCE_ACCOUNT_LENGTH,
+);
+
+// Form CreateNonceAccount transaction
+let transaction = new web3.Transaction().add(
+web3.SystemProgram.createNonceAccount({
+    fromPubkey: account.publicKey,
+    noncePubkey: nonceAccount.publicKey,
+    authorizedPubkey: account.publicKey,
+    lamports: minimumAmount,
+}),
+);
+// Create Nonce Account
+await web3.sendAndConfirmTransaction(
+    connection,
+    transaction,
+    [account, nonceAccount]
+);
+
+let nonceAccountData = await connection.getNonce(
+    nonceAccount.publicKey,
+    'confirmed',
+);
+
+console.log(nonceAccountData);
+// NonceAccount {
+//   authorizedPubkey: PublicKey {
+//     _bn: <BN: 919981a5497e8f85c805547439ae59f607ea625b86b1138ea6e41a68ab8ee038>
+//   },
+//   nonce: '93zGZbhMmReyz4YHXjt2gHsvu5tjARsyukxD4xnaWaBq',
+//   feeCalculator: { lamportsPerSignature: 5000 }
+// }
+
+let nonceAccountInfo = await connection.getAccountInfo(
+    nonceAccount.publicKey,
+    'confirmed'
+);
+
+let nonceAccountFromInfo = web3.NonceAccount.fromAccountData(
+    nonceAccountInfo.data
+);
+
+console.log(nonceAccountFromInfo);
+// NonceAccount {
+//   authorizedPubkey: PublicKey {
+//     _bn: <BN: 919981a5497e8f85c805547439ae59f607ea625b86b1138ea6e41a68ab8ee038>
+//   },
+//   nonce: '93zGZbhMmReyz4YHXjt2gHsvu5tjARsyukxD4xnaWaBq',
+//   feeCalculator: { lamportsPerSignature: 5000 }
+// }
+```
+
+The above example shows both how to create a `NonceAccount` using `SystemProgram.createNonceAccount`, as well as how to retrieve the `NonceAccount` from accountInfo. Using the nonce, you can create transactions offline with the nonce in place of the `recentBlockhash`.
+
+### VoteAccount
+
+[SourceDocumentation](https://solana-labs.github.io/solana-web3.js/classes/VoteAccount.html)
+
+Vote account is an object that grants the capability of decoding vote accounts from the native vote account program on the network.
+
+#### Example Usage
+
+```javascript
+const web3 = require('@solana/web3.js');
+
+let voteAccountInfo = await connection.getProgramAccounts(web3.VOTE_PROGRAM_ID);
+let voteAccountFromData = web3.VoteAccount.fromAccountData(voteAccountInfo[0].account.data);
+console.log(voteAccountFromData);
+/*
+VoteAccount {
+    nodePubkey: PublicKey {
+        _bn: <BN: 100000096c4e1e61a6393e51937e548aee2c062e23c67cbaa8d04f289d18232>
+    },
+    authorizedVoterPubkey: PublicKey {
+        _bn: <BN: 5862b94396c4e1e61a6393e51937e548aee2c062e23c67cbaa8d04f289d18232>
+    },
+    authorizedWithdrawerPubkey: PublicKey {
+        _bn: <BN: 5862b9430a0800000000000000cb136204000000001f000000cc136204000000>
+    },
+    commission: 0,
+    votes: [
+        { slot: 124554051584, confirmationCount: 73536462 },
+        { slot: 120259084288, confirmationCount: 73536463 },
+        { slot: 115964116992, confirmationCount: 73536464 },
+        { slot: 111669149696, confirmationCount: 73536465 },
+        { slot: 107374182400, confirmationCount: 96542804 },
+        { slot: 4294967296, confirmationCount: 1645464065 },
+        { slot: 1099511627780, confirmationCount: 0 },
+        { slot: 57088, confirmationCount: 3787757056 },
+        { slot: 16516698632215534000, confirmationCount: 3236081224 },
+        { slot: 328106138455040640, confirmationCount: 2194770418 },
+        { slot: 290873038898, confirmationCount: 0 },
+    ],
+    rootSlot: null,
+    epoch: 0,
+    credits: 0,
+    lastEpochCredits: 0,
+    epochCredits: []
+}
+*/
+```
+
+## Staking
+
+### StakeProgram
+
+[SourceDocumentation](https://solana-labs.github.io/solana-web3.js/classes/StakeProgram.html)
+
+The StakeProgram facilitates staking SOL and delegating them to any validators on the network. You can use StakeProgram to create a stake account, stake some SOL, authorize accounts for withdrawal of your stake, deactivate your stake, and withdraw your funds. The StakeInstruction class is used to decode and read more instructions from transactions calling the StakeProgram
+
+#### Example Usage
+
+```javascript
+const web3 = require("@solana/web3.js");
+
+// Fund a key to create transactions
+let fromPublicKey = web3.Keypair.generate();
+let connection = new web3.Connection(web3.clusterApiUrl('devnet'), 'confirmed');
+
+let airdropSignature = await connection.requestAirdrop(
+    fromPublicKey.publicKey,
+    web3.LAMPORTS_PER_SOL,
+);
+await connection.confirmTransaction(airdropSignature);
+
+// Create Account
+let stakeAccount = web3.Keypair.generate();
+let authorizedAccount = web3.Keypair.generate();
+/* Note: This is the minimum amount for a stake account -- Add additional Lamports for staking
+    For example, we add 50 lamports as part of the stake */
+let lamportsForStakeAccount = (await connection.getMinimumBalanceForRentExemption(web3.StakeProgram.space)) + 50;
+
+let createAccountTransaction = web3.StakeProgram.createAccount({
+    fromPubkey: fromPublicKey.publicKey,
+    authorized: new web3.Authorized(authorizedAccount.publicKey, authorizedAccount.publicKey),
+    lamports: lamportsForStakeAccount,
+    lockup: new web3.Lockup(0, 0, fromPublicKey.publicKey),
+    stakePubkey: stakeAccount.publicKey
+});
+await web3.sendAndConfirmTransaction(connection, createAccountTransaction, [fromPublicKey, stakeAccount]);
+
+// Check that stake is available
+let stakeBalance = await connection.getBalance(stakeAccount.publicKey);
+console.log(`Stake balance: ${stakeBalance}`)
+// Stake balance: 2282930
+
+// We can verify the state of our stake. This may take some time to become active
+let stakeState = await connection.getStakeActivation(stakeAccount.publicKey);
+console.log(`Stake Stake: ${stakeState.state}`);
+// Stake State: inactive
+
+// To delegate our stake, we get the current vote accounts and choose the first
+let voteAccounts = await connection.getVoteAccounts();
+let voteAccount = voteAccounts.current.concat(
+    voteAccounts.delinquent,
+)[0];
+let votePubkey = new web3.PublicKey(voteAccount.votePubkey);
+
+// We can then delegate our stake to the voteAccount
+let delegateTransaction = web3.StakeProgram.delegate({
+    stakePubkey: stakeAccount.publicKey,
+    authorizedPubkey: authorizedAccount.publicKey,
+    votePubkey: votePubkey,
+});
+await web3.sendAndConfirmTransaction(connection, delegateTransaction, [fromPublicKey, authorizedAccount]);
+
+// To withdraw our funds, we first have to deactivate the stake
+let deactivateTransaction = web3.StakeProgram.deactivate({
+    stakePubkey: stakeAccount.publicKey,
+    authorizedPubkey: authorizedAccount.publicKey,
+});
+await web3.sendAndConfirmTransaction(connection, deactivateTransaction, [fromPublicKey, authorizedAccount]);
+
+// Once deactivated, we can withdraw our funds
+let withdrawTransaction = web3.StakeProgram.withdraw({
+    stakePubkey: stakeAccount.publicKey,
+    authorizedPubkey: authorizedAccount.publicKey,
+    toPubkey: fromPublicKey.publicKey,
+    lamports: stakeBalance,
+});
+
+await web3.sendAndConfirmTransaction(connection, withdrawTransaction, [fromPublicKey, authorizedAccount]);
+```
+
+### Authorized
+
+[Source Documentation](https://solana-labs.github.io/solana-web3.js/classes/Authorized.html)
+
+Authorized is an object used when creating an authorized account for staking within Solana. You can designate a `staker` and `withdrawer` separately, allowing for a different account to withdraw other than the staker.
+
+You can find more usage of the `Authorized` object under [`StakeProgram`](https://solana-labs.github.io/solana-web3.js/classes/StakeProgram.html)
+
+### Lockup
+
+[Source Documentation](https://solana-labs.github.io/solana-web3.js/classes/Lockup.html)
+
+Lockup is used in conjunction with the [StakeProgram](https://solana-labs.github.io/solana-web3.js/classes/StakeProgram.html) to create an account. The Lockup is used to determine how long the stake will be locked, or unable to be retrieved. If the Lockup is set to 0 for both epoch and the Unix timestamp, the lockup will be disabled for the stake account.
+
+#### Example Usage
+
+```javascript
+const {Authorized, Keypair, Lockup, StakeProgram} = require("@solana/web3.js");
+
+let account = Keypair.generate();
+let stakeAccount = Keypair.generate();
+let authorized = new Authorized(account.publicKey, account.publicKey);
+let lockup = new Lockup(0, 0, account.publicKey);
+
+let createStakeAccountInstruction = StakeProgram.createAccount({
+    fromPubkey: account.publicKey,
+    authorized: authorized,
+    lamports: 1000,
+    lockup: lockup,
+    stakePubkey: stakeAccount.publicKey
+});
+```
+The above code creates a `createStakeAccountInstruction` to be used when creating an account with the `StakeProgram`. The Lockup is set to 0 for both the epoch and Unix timestamp, disabling lockup for the account.
+
+See [StakeProgram](https://solana-labs.github.io/solana-web3.js/classes/StakeProgram.html) for more.

docs/src/developing/clients/jsonrpc-api.md

@@ -1340,7 +1340,7 @@ The result field will be a JSON object with the following fields:
 - `total: <f64>`, total inflation
 - `validator: <f64>`, inflation allocated to validators
 - `foundation: <f64>`, inflation allocated to the foundation
-- `epoch: <f64>`, epoch for which these values are valid
+- `epoch: <u64>`, epoch for which these values are valid
 
 #### Example:
 
@@ -3039,7 +3039,7 @@ curl http://localhost:8899 -X POST -H "Content-Type: application/json" -d '
 
 Result:
 ```json
-{"jsonrpc":"2.0","result":{"solana-core": "1.7.15"},"id":1}
+{"jsonrpc":"2.0","result":{"solana-core": "1.8.1"},"id":1}
 ```
 
 ### getVoteAccounts
@@ -4151,7 +4151,7 @@ Response:
 ### getConfirmedBlock
 
 **DEPRECATED: Please use [getBlock](jsonrpc-api.md#getblock) instead**
-This method is expected to be removed in solana-core v1.8
+This method is expected to be removed in solana-core v2.0
 
 Returns identity and transaction information about a confirmed block in the ledger
 
@@ -4345,7 +4345,7 @@ For more details on returned data:
 ### getConfirmedBlocks
 
 **DEPRECATED: Please use [getBlocks](jsonrpc-api.md#getblocks) instead**
-This method is expected to be removed in solana-core v1.8
+This method is expected to be removed in solana-core v2.0
 
 Returns a list of confirmed blocks between two slots
 
@@ -4379,7 +4379,7 @@ Result:
 ### getConfirmedBlocksWithLimit
 
 **DEPRECATED: Please use [getBlocksWithLimit](jsonrpc-api.md#getblockswithlimit) instead**
-This method is expected to be removed in solana-core v1.8
+This method is expected to be removed in solana-core v2.0
 
 Returns a list of confirmed blocks starting at the given slot
 
@@ -4411,7 +4411,7 @@ Result:
 ### getConfirmedSignaturesForAddress2
 
 **DEPRECATED: Please use [getSignaturesForAddress](jsonrpc-api.md#getsignaturesforaddress) instead**
-This method is expected to be removed in solana-core v1.8
+This method is expected to be removed in solana-core v2.0
 
 Returns confirmed signatures for transactions involving an
 address backwards in time from the provided signature or most recent confirmed block
@@ -4473,7 +4473,7 @@ Result:
 ### getConfirmedTransaction
 
 **DEPRECATED: Please use [getTransaction](jsonrpc-api.md#gettransaction) instead**
-This method is expected to be removed in solana-core v1.8
+This method is expected to be removed in solana-core v2.0
 
 Returns transaction details for a confirmed transaction
 

docs/src/developing/clients/rust-api.md

@@ -2,7 +2,30 @@
 title: Rust API
 ---
 
-See [doc.rs](https://docs.rs/releases/search?query=solana-) for documentation of
-all crates published by Solana. In particular [solana-sdk](https://docs.rs/solana-sdk)
-for working with common data structures and [solana-client](https://docs.rs/solana-client)
-for querying the [JSON RPC API](jsonrpc-api).
+Solana's Rust crates are [published to crates.io][crates.io] and can be found
+[on docs.rs with the "solana-" prefix][docs.rs].
+
+[crates.io]: https://crates.io/search?q=solana-
+[docs.rs]: https://docs.rs/releases/search?query=solana-
+
+Some important crates:
+
+- [`solana-program`] — Imported by programs running on Solana, compiled
+  to BPF. This crate contains many fundamental data types and is re-exported from
+  [`solana-sdk`], which cannot be imported from a Solana program.
+
+- [`solana-sdk`] — The basic off-chain SDK, it re-exports
+  [`solana-program`] and adds more APIs on top of that. Most Solana programs
+  that do not run on-chain will import this.
+
+- [`solana-client`] — For interacting with a Solana node via the
+  [JSON RPC API](jsonrpc-api).
+
+- [`solana-clap-utils`] — Routines for setting up a CLI, using [`clap`],
+  as used by the main Solana CLI.
+
+[`solana-program`]: https://docs.rs/solana-program
+[`solana-sdk`]: https://docs.rs/solana-sdk
+[`solana-client`]: https://docs.rs/solana-client
+[`solana-clap-utils`]: https://docs.rs/solana-clap-utils
+[`clap`]: https://docs.rs/clap

docs/src/developing/runtime-facilities/programs.md

@@ -65,6 +65,51 @@ to the BPF Upgradeable Loader to process the instruction.
 
 [More information about deployment](cli/deploy-a-program.md)
 
+## Ed25519 Program
+
+Verify ed25519 signature program. This program takes an ed25519 signature, public key, and message.
+Multiple signatures can be verified. If any of the signatures fail to verify, an error is returned.
+
+- Program id: `Ed25519SigVerify111111111111111111111111111`
+- Instructions: [new_ed25519_instruction](https://github.com/solana-labs/solana/blob/master/sdk/src/ed25519_instruction.rs#L45)
+
+The ed25519 program processes an instruction. The first `u8` is a count of the number of
+signatures to check, which is followed by a single byte padding. After that, the
+following struct is serialized, one for each signature to check.
+
+```
+struct Ed25519SignatureOffsets {
+    signature_offset: u16,             // offset to ed25519 signature of 64 bytes
+    signature_instruction_index: u16,  // instruction index to find signature
+    public_key_offset: u16,            // offset to public key of 32 bytes
+    public_key_instruction_index: u16, // instruction index to find public key
+    message_data_offset: u16,          // offset to start of message data
+    message_data_size: u16,            // size of message data
+    message_instruction_index: u16,    // index of instruction data to get message data
+}
+```
+
+Pseudo code of the operation:
+
+```
+process_instruction() {
+    for i in 0..count {
+        // i'th index values referenced:
+        instructions = &transaction.message().instructions
+        instruction_index = ed25519_signature_instruction_index != u16::MAX ? ed25519_signature_instruction_index : current_instruction;
+        signature = instructions[instruction_index].data[ed25519_signature_offset..ed25519_signature_offset + 64]
+        instruction_index = ed25519_pubkey_instruction_index != u16::MAX ? ed25519_pubkey_instruction_index : current_instruction;
+        pubkey = instructions[instruction_index].data[ed25519_pubkey_offset..ed25519_pubkey_offset + 32]
+        instruction_index = ed25519_message_instruction_index != u16::MAX ? ed25519_message_instruction_index : current_instruction;
+        message = instructions[instruction_index].data[ed25519_message_data_offset..ed25519_message_data_offset + ed25519_message_data_size]
+        if pubkey.verify(signature, message) != Success {
+            return Error
+        }
+    }
+    return Success
+}
+```
+
 ## Secp256k1 Program
 
 Verify secp256k1 public key recovery operations (ecrecover).

docs/src/implemented-proposals/rpc-transaction-history.md

@@ -16,10 +16,7 @@ The affected RPC endpoints are:
 - [getConfirmedSignaturesForAddress](developing/clients/jsonrpc-api.md#getconfirmedsignaturesforaddress)
 - [getConfirmedTransaction](developing/clients/jsonrpc-api.md#getconfirmedtransaction)
 - [getSignatureStatuses](developing/clients/jsonrpc-api.md#getsignaturestatuses)
-
-Note that [getBlockTime](developing/clients/jsonrpc-api.md#getblocktime)
-is not supported, as once https://github.com/solana-labs/solana/issues/10089 is
-fixed then `getBlockTime` can be removed.
+- [getBlockTime](developing/clients/jsonrpc-api.md#getblocktime)
 
 Some system design constraints:
 

docs/src/proposals/log_data.md

@@ -0,0 +1,146 @@
+# Program log binary data
+
+## Problem
+
+There is no support for logging binary data in Solidity.
+
+### Events in Solidity
+
+In Solidity, events can be reported. These look like structures with zero or
+more fields, and can be emitted with specific values. For example:
+
+```
+event PaymentReceived {
+    address sender;
+    uint amount;
+}
+
+contract c {
+    function pay() public payable {
+        emit PaymentReceived(msg.sender, msg.value);
+    }
+}
+```
+
+Events are write-only from a Solidity/VM perspective and are written to
+the blocks in the tx records.
+
+Some of these fields can be marked `indexed`, which affects how the data is
+encoded. All non-indexed fields are eth abi encoded into a variable length
+byte array. All indexed fields go into so-called topics.
+
+Topics are fixed length fields of 32 bytes. There are a maximum of 4 topics;
+if a type does not always fit into 32 bytes (e.g. string types), then the topic
+is keccak256 hashed.
+
+The first topic is a keccak256 hash of the event signature, in this case
+`keccak256('PaymentReceived(address,uint)')`. The four remaining are available
+for `indexed` fields. The event may be declared `anonymous`, in which case
+the first field is not a hash of the signature, and it is permitted to have
+4 indexed fields.
+
+### Listening to events in a client
+
+The reason for the distinction between topics/indexed and regular fields is
+that it easier to filter on topics.
+
+```
+const Web3 = require('web3');
+const url = 'ws://127.0.0.1:8546';
+const web3 = new Web3(url);
+
+var options = {
+    address: '0xfbBE8f06FAda977Ea1E177da391C370EFbEE3D25',
+    topics: [
+        '0xdf50c7bb3b25f812aedef81bc334454040e7b27e27de95a79451d663013b7e17',
+        //'0x0000000000000000000000000d8a3f5e71560982fb0eb5959ecf84412be6ae3e'
+      ]
+};
+
+var subscription = web3.eth.subscribe('logs', options, function(error, result){
+    if (!error) console.log('got result');
+    else console.log(error);
+}).on("data", function(log){
+    console.log('got data', log);
+}).on("changed", function(log){
+    console.log('changed');
+});
+```
+
+In order to decode the non-indexed fields (the data), the abi of the contract
+is needed. So, the topic is first used to discover what event was used, and
+then the data can be decoded.
+
+### Ethereum Tx in block
+
+The transaction calls event logs. Here is a tx with a single event, with 3
+topics and some data.
+
+```
+{
+  "tx": {
+    "nonce": "0x2",
+    "gasPrice": "0xf224d4a00",
+    "gas": "0xc350",
+    "to": "0x6B175474E89094C44Da98b954EedeAC495271d0F",
+    "value": "0x0",
+    "input": "0xa9059cbb000000000000000000000000a12431d0b9db640034b0cdfceef9cce161e62be40000000000000000000000000000000000000000000000a030dcebbd2f4c0000",
+    "hash": "0x98a67f0a35ebc0ac068acf0885d38419c632ffa4354e96641d6d5103a7681910",
+    "blockNumber": "0xc96431",
+    "from": "0x82f890D638478d211eF2208f3c1466B5Abf83551",
+    "transactionIndex": "0xe1"
+  },
+  "receipt": {
+    "gasUsed": "0x74d2",
+    "status": "0x1",
+    "logs": [
+      {
+        "address": "0x6B175474E89094C44Da98b954EedeAC495271d0F",
+        "topics": [
+          "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
+          "0x00000000000000000000000082f890d638478d211ef2208f3c1466b5abf83551",
+          "0x000000000000000000000000a12431d0b9db640034b0cdfceef9cce161e62be4"
+        ],
+        "data": "0x0000000000000000000000000000000000000000000000a030dcebbd2f4c0000"
+      }
+    ]
+  }
+}
+```
+
+### Further considerations
+
+In Ethereum, events are stored in blocks. Events mark certain state changes in
+smart contracts. This serves two purposes:
+
+ - Listen to events (i.e. state changes) as they happen by reading new blocks
+   as they are published
+ - Re-read historical events by reading old blocks
+
+So for example, smart contracts may emit changes as they happen but never the
+complete state, so the only way to recover the entire state of the contract
+is by re-reading all events from the chain. So an application will read events
+from block 1 or whatever block the application was deployed at and then use
+that state for local processing. This is a local cache and may re-populated
+from the chain at any point.
+
+## Proposed Solution
+
+Binary logging should be added to the program log. The program log should include the base64 encoded data (zero or more one permitted).
+
+So if we encoding the topics first, followed by the data then the event in the
+tx above would look like:
+```
+program data: 3fJSrRviyJtpwrBo/DeNqpUrpFjxKEWKPVaTfUjs8AAAAAAAAAAAAAAACC+JDWOEeNIR7yII88FGa1q/g1UQAAAAAAAAAAAAAAAKEkMdC522QANLDN/O75zOFh5ivk AAAAAAAAAAAAAAAAAAAAAAAAAAAAAACgMNzrvS9MAAA=
+```
+
+This requires a new system call:
+
+```
+void sol_log_data(SolBytes *fields, uint64_t length);
+```
+
+### Considerations
+
+- Should there be text field in the program log so we can have a little bit of
+  metadata on the binary data, to make it more human readable

docs/src/proposals/optimistic-confirmation-and-slashing.md

@@ -39,9 +39,9 @@ on our mainnet, cosmos, or tezos. For our network, which is primarily
 composed of high availability systems, this seems unlikely. Currently,
 we have set the threshold percentage to 4.66%, which means that if 23.68%
 have failed the network may stop finalizing blocks. For our network,
-which is primarily composed of high availability systems a 23.68% drop
-in availabilty seems unlinkely. 1:10^12 odds assuming five 4.7% staked
-nodes with 0.995 of uptime.
+which is primarily composed of high availability systems, a 23.68% drop
+in availability seems unlikely. 1:10^12 odds, assuming five 4.7% staked
+nodes with 0.995 uptime.
 
 ## Security
 

docs/src/proposals/return-data.md

@@ -0,0 +1,144 @@
+# Return data from BPF programs
+
+## Problem
+
+In the Solidity langauge it is permitted to return any number of values from a function,
+for example a variable length string can be returned:
+
+```
+function foo1() public returns (string) {
+    return "Hello, world!\n";
+}
+```
+
+Multiple values, arrays and structs are permitted too.
+
+```
+struct S {
+    int f1;
+    bool f2
+};
+
+function foo2() public returns (string, int[], S) {
+    return (a, b, c);
+}
+```
+
+All the return values are eth abi encoded to a variable-length byte array.
+
+On ethereum errors can be returned too:
+
+```
+function withdraw() public {
+    require(msg.sender == owner, "Permission denied");
+}
+
+function failure() public {
+    revert("I afraid I can't do that dave");
+}
+```
+These errors help the developer debug any issue they are having, and can
+also be caught in a Solidity `try` .. `catch` block. Outside of a `try` .. `catch`
+block, any of these would cause the transaction or rpc to fail.
+
+## Existing solution
+
+The existing solution that Solang uses, writes the return data to the callee account data.
+The caller's account cannot be used, since the callee may not be the same BPF program, so
+it will not have permission to write to the callee's account data.
+
+Another solution would be to have a single return data account which is passed
+around through CPI. Again this does not work for CPI as the callee may not have
+permission to write to it.
+
+The problem with this solution is:
+
+- It does not work for RPC calls
+- It is very racey; a client has to submit the Tx and then retrieve the account
+  data. This is not atomic so the return data can be overwritten by another transaction.
+
+## Requirements for Solution
+
+It must work for:
+
+- RPC: An RPC should be able to return any number of values without writing to account data
+- Transaction: An transaction should be able to return any number of values without needing to write them account data
+- CPI: The callee must "set" return value, and the caller must be able to retrieve it.
+
+## Review of other chains
+
+### Ethereum (EVM)
+
+The `RETURN` opcode allows a contract to set a buffer as a returndata. This opcode takes a pointer to memory and a size. The `REVERT` opcode works similarly but signals that the call failed, and all account data changes must be reverted.
+
+For CPI, the caller can retrieve the returned data of the callee using the `RETURNDATASIZE` opcode which returns the length, and the `RETURNDATACOPY` opcode, which takes a memory destination pointer, offset into the returndata, and a length argument.
+
+Ethereum stores the returndata in blocks.
+
+### Parity Substrate
+
+The return data can be set using the `seal_return(u32 flags, u32 pointer, u32 size)` syscall.
+- Flags can be 1 for revert, 0 for success (nothing else defined)
+- Function does not return
+
+CPI: The `seal_call()` syscall takes pointer to buffer and pointer to buffer size where return data goes
+ - There is a 32KB limit for return data.
+
+Parity Substrate does not write the return data to blocks.
+
+## Rejected Solution
+
+The concept of ephemeral accounts has been proposed a solution for this. This would
+certainly work for the CPI case, but this would not work RPC or Transaction case.
+
+## Proposed Solution
+
+The callee can set the return data using a new system call `sol_set_return_data(buf: *const u8, length: u64)`.
+There is a limit of 1024 bytes for the returndata. This function can be called multiple times, and
+will simply overwrite what was written in the last call.
+
+The return data can be retrieved with `sol_get_return_data(buf: *mut u8, length: u64, program_id: *mut Pubkey) -> u64`.
+This function copies the return buffer, and the program_id that set the return data, and
+returns the length of the return data, or `0` if no return data is set. In this case, program_id is not set.
+
+When an instruction calls `sol_invoke()`, the return data of the callee is copied into the return data
+of the current instruction. This means that any return data is automatically passed up the call stack,
+to the callee of the current instruction (or the RPC call).
+
+Note that `sol_invoke()` clears the returns data before invoking the callee, so that any return data from
+a previous invoke is not reused if the invoked fails to set a return data. For example:
+
+ - A invokes B
+ - Before entry to B, return data is cleared.0
+ - B sets some return data and returns
+ - A invokes C
+ - Before entry to C, return data is cleared.
+ - C does not set return data and returns
+ - A checks return data and finds it empty
+
+Another scenario to consider:
+
+ - A invokes B
+ - B invokes C
+ - C sets return data and returns
+ - B does not touch return data and returns
+ - A gets return data from C
+ - A does not touch return data
+ - Return data from transaction is what C set.
+
+The compute costs are calculated for getting and setting the return data using
+the syscalls.
+
+For a normal RPC or Transaction, the returndata is base64-encoded and stored along side the sol_log
+strings in the [stable log](https://github.com/solana-labs/solana/blob/95292841947763bdd47ef116b40fc34d0585bca8/sdk/src/process_instruction.rs#L275-L281).
+
+## Note on returning errors
+
+Solidity on Ethereum allows the contract to return an error in the return data. In this case, all
+the account data changes for the account should be reverted. On Solana, any non-zero exit code
+for a BPF prorgram means the entire transaction fails. We do not wish to support an error return
+by returning success and then returning an error in the return data. This would mean we would have
+to support reverting the account data changes; this too expensive both on the VM side and the BPF
+contract side.
+
+Errors will be reported via sol_log.

docs/src/running-validator/validator-monitor.md

@@ -8,7 +8,7 @@ Confirm the IP address and **identity pubkey** of your validator is visible in
 the gossip network by running:
 
 ```bash
-solana-gossip spy --entrypoint devnet.solana.com:8001
+solana gossip
 ```
 
 ## Check Your Balance

docs/src/running-validator/validator-start.md

@@ -26,16 +26,6 @@ solana transaction-count
 View the [metrics dashboard](https://metrics.solana.com:3000/d/monitor/cluster-telemetry) for more
 detail on cluster activity.
 
-## Confirm your Installation
-
-Try running following command to join the gossip network and view all the other
-nodes in the cluster:
-
-```bash
-solana-gossip spy --entrypoint entrypoint.devnet.solana.com:8001
-# Press ^C to exit
-```
-
 ## Enabling CUDA
 
 If your machine has a GPU with CUDA installed \(Linux-only currently\), include
@@ -318,11 +308,11 @@ The ledger will be placed in the `ledger/` directory by default, use the
 > `solana-validator --identity ASK ... --authorized-voter ASK ...`
 > and you will be prompted to enter your seed phrases and optional passphrase.
 
-Confirm your validator connected to the network by opening a new terminal and
+Confirm your validator is connected to the network by opening a new terminal and
 running:
 
 ```bash
-solana-gossip spy --entrypoint entrypoint.devnet.solana.com:8001
+solana gossip
 ```
 
 If your validator is connected, its public key and IP address will appear in the list.

dos/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-dos"
-version = "1.7.15"
+version = "1.8.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,18 +14,18 @@ clap = "2.33.1"
 log = "0.4.11"
 rand = "0.7.0"
 rayon = "1.5.0"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-core = { path = "../core", version = "=1.7.15" }
-solana-gossip = { path = "../gossip", version = "=1.7.15" }
-solana-ledger = { path = "../ledger", version = "=1.7.15" }
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.15" }
-solana-perf = { path = "../perf", version = "=1.7.15" }
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-streamer = { path = "../streamer", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
-solana-client = { path = "../client", version = "=1.7.15" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-core = { path = "../core", version = "=1.8.1" }
+solana-gossip = { path = "../gossip", version = "=1.8.1" }
+solana-ledger = { path = "../ledger", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.1" }
+solana-perf = { path = "../perf", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-streamer = { path = "../streamer", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
+solana-client = { path = "../client", version = "=1.8.1" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

download-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-download-utils"
-version = "1.7.15"
+version = "1.8.1"
 description = "Solana Download Utils"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -15,8 +15,8 @@ console = "0.14.1"
 indicatif = "0.15.0"
 log = "0.4.11"
 reqwest = { version = "0.11.2", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
 tar = "0.4.37"
 
 [lib]

faucet/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-faucet"
-version = "1.7.15"
+version = "1.8.1"
 description = "Solana Faucet"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -16,12 +16,12 @@ clap = "2.33"
 log = "0.4.11"
 serde = "1.0.122"
 serde_derive = "1.0.103"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-cli-config = { path = "../cli-config", version = "=1.7.15" }
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-metrics = { path = "../metrics", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-cli-config = { path = "../cli-config", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-metrics = { path = "../metrics", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }

frozen-abi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-frozen-abi"
-version = "1.7.15"
+version = "1.8.1"
 description = "Solana Frozen ABI"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -16,11 +16,11 @@ log = "0.4.11"
 serde = "1.0.122"
 serde_derive = "1.0.103"
 sha2 = "0.9.2"
-solana-frozen-abi-macro = { path = "macro", version = "=1.7.15" }
+solana-frozen-abi-macro = { path = "macro", version = "=1.8.1" }
 thiserror = "1.0"
 
 [target.'cfg(not(target_arch = "bpf"))'.dependencies]
-solana-logger = { path = "../logger", version = "=1.7.15" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
 generic-array = { version = "0.14.3", default-features = false, features = ["serde", "more_lengths"]}
 memmap2 = "0.1.0"
 

frozen-abi/macro/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-frozen-abi-macro"
-version = "1.7.15"
+version = "1.8.1"
 description = "Solana Frozen ABI Macro"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

genesis-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-genesis-utils"
-version = "1.7.15"
+version = "1.8.1"
 description = "Solana Genesis Utils"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,9 +10,9 @@ documentation = "https://docs.rs/solana-download-utils"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-download-utils = { path = "../download-utils", version = "=1.7.15" }
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-download-utils = { path = "../download-utils", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
 
 [lib]
 crate-type = ["lib"]

genesis/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-genesis"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.15"
+version = "1.8.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -16,16 +16,16 @@ chrono = "0.4"
 serde = "1.0.122"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-cli-config = { path = "../cli-config", version = "=1.7.15" }
-solana-exchange-program = { path = "../programs/exchange", version = "=1.7.15" }
-solana-ledger = { path = "../ledger", version = "=1.7.15" }
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-stake-program = { path = "../programs/stake", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.15" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-cli-config = { path = "../cli-config", version = "=1.8.1" }
+solana-exchange-program = { path = "../programs/exchange", version = "=1.8.1" }
+solana-ledger = { path = "../ledger", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-stake-program = { path = "../programs/stake", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.8.1" }
 tempfile = "3.1.0"
 
 [[bin]]

gossip/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-gossip"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.15"
+version = "1.8.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -26,22 +26,22 @@ rayon = "1.5.0"
 serde = "1.0.122"
 serde_bytes = "0.11"
 serde_derive = "1.0.103"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-client = { path = "../client", version = "=1.7.15" }
-solana-frozen-abi = { path = "../frozen-abi", version = "=1.7.15" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.7.15" }
-solana-ledger = { path = "../ledger", version = "=1.7.15" }
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-measure = { path = "../measure", version = "=1.7.15" }
-solana-metrics = { path = "../metrics", version = "=1.7.15" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.15" }
-solana-perf = { path = "../perf", version = "=1.7.15" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.7.15" }
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
-solana-streamer = { path = "../streamer", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.15" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-client = { path = "../client", version = "=1.8.1" }
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.8.1" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.8.1" }
+solana-ledger = { path = "../ledger", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-measure = { path = "../measure", version = "=1.8.1" }
+solana-metrics = { path = "../metrics", version = "=1.8.1" }
+solana-net-utils = { path = "../net-utils", version = "=1.8.1" }
+solana-perf = { path = "../perf", version = "=1.8.1" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
+solana-streamer = { path = "../streamer", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.8.1" }
 thiserror = "1.0"
 
 [dev-dependencies]

gossip/src/cluster_info_metrics.rs

@@ -163,11 +163,12 @@ pub(crate) fn submit_gossip_stats(
     gossip: &RwLock<CrdsGossip>,
     stakes: &HashMap<Pubkey, u64>,
 ) {
-    let (table_size, num_nodes, purged_values_size, failed_inserts_size) = {
+    let (table_size, num_nodes, num_pubkeys, purged_values_size, failed_inserts_size) = {
         let gossip = gossip.read().unwrap();
         (
             gossip.crds.len(),
             gossip.crds.num_nodes(),
+            gossip.crds.num_pubkeys(),
             gossip.crds.num_purged(),
             gossip.pull.failed_inserts.len(),
         )
@@ -192,6 +193,7 @@ pub(crate) fn submit_gossip_stats(
         ("failed_inserts_size", failed_inserts_size as i64, i64),
         ("num_nodes", num_nodes as i64, i64),
         ("num_nodes_staked", num_nodes_staked as i64, i64),
+        ("num_pubkeys", num_pubkeys, i64),
     );
     datapoint_info!(
         "cluster_info_stats2",

gossip/src/crds.rs

@@ -319,8 +319,13 @@ impl Crds {
             .map(move |i| self.table.index(*i))
     }
 
-    /// Returns number of known pubkeys (network size).
+    /// Returns number of known contact-infos (network size).
     pub(crate) fn num_nodes(&self) -> usize {
+        self.nodes.len()
+    }
+
+    /// Returns number of unique pubkeys.
+    pub(crate) fn num_pubkeys(&self) -> usize {
         self.records.len()
     }
 

install/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-install"
 description = "The solana cluster software installer"
-version = "1.7.15"
+version = "1.8.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -16,21 +16,21 @@ bzip2 = "0.3.3"
 chrono = { version = "0.4.11", features = ["serde"] }
 clap = { version = "2.33.1" }
 console = "0.14.1"
-ctrlc = { version = "3.1.5", features = ["termination"] }
+ctrlc = { version = "3.1.9", features = ["termination"] }
 dirs-next = "2.0.0"
 indicatif = "0.15.0"
 lazy_static = "1.4.0"
-nix = "0.19.0"
+nix = "0.20.0"
 reqwest = { version = "0.11.2", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 serde = { version = "1.0.122", features = ["derive"] }
 serde_json = "1.0.62"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-client = { path = "../client", version = "=1.7.15" }
-solana-config-program = { path = "../programs/config", version = "=1.7.15" }
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-client = { path = "../client", version = "=1.8.1" }
+solana-config-program = { path = "../programs/config", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
 semver = "0.9.0"
 tar = "0.4.37"
 tempfile = "3.1.0"

keygen/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-keygen"
-version = "1.7.15"
+version = "1.8.1"
 description = "Solana key generation utility"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,11 +14,11 @@ bs58 = "0.3.1"
 clap = "2.33"
 dirs-next = "2.0.0"
 num_cpus = "1.13.0"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-cli-config = { path = "../cli-config", version = "=1.7.15" }
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-cli-config = { path = "../cli-config", version = "=1.8.1" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
 tiny-bip39 = "0.8.1"
 
 [[bin]]

ledger-tool/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-ledger-tool"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.7.15"
+version = "1.8.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -24,18 +24,19 @@ regex = "1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.15" }
-solana-cli-output = { path = "../cli-output", version = "=1.7.15" }
-solana-ledger = { path = "../ledger", version = "=1.7.15" }
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-measure = { path = "../measure", version = "=1.7.15" }
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-stake-program = { path = "../programs/stake", version = "=1.7.15" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.7.15" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.15" }
-solana-version = { path = "../version", version = "=1.7.15" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.15" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.8.1" }
+solana-cli-output = { path = "../cli-output", version = "=1.8.1" }
+solana-core = { path = "../core", version = "=1.8.1" }
+solana-ledger = { path = "../ledger", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-measure = { path = "../measure", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-stake-program = { path = "../programs/stake", version = "=1.8.1" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.8.1" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.8.1" }
+solana-version = { path = "../version", version = "=1.8.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.8.1" }
 tempfile = "3.1.0"
 tokio = { version = "1", features = ["full"] }
 

ledger-tool/src/main.rs

@@ -27,6 +27,9 @@ use solana_ledger::{
 use solana_runtime::{
     bank::{Bank, RewardCalculationEvent},
     bank_forks::{ArchiveFormat, BankForks, SnapshotConfig},
+    cost_model::CostModel,
+    cost_tracker::CostTracker,
+    cost_tracker_stats::CostTrackerStats,
     hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
     snapshot_utils,
     snapshot_utils::{SnapshotVersion, DEFAULT_MAX_SNAPSHOTS_TO_RETAIN},
@@ -724,9 +727,64 @@ fn load_bank_forks(
         process_options,
         None,
         None,
+        None,
     )
 }
 
+fn compute_slot_cost(blockstore: &Blockstore, slot: Slot) -> Result<(), String> {
+    if blockstore.is_dead(slot) {
+        return Err("Dead slot".to_string());
+    }
+
+    let (entries, _num_shreds, _is_full) = blockstore
+        .get_slot_entries_with_shred_info(slot, 0, false)
+        .map_err(|err| format!(" Slot: {}, Failed to load entries, err {:?}", slot, err))?;
+
+    let mut transactions = 0;
+    let mut programs = 0;
+    let mut program_ids = HashMap::new();
+    let mut cost_model = CostModel::default();
+    cost_model.initialize_cost_table(&blockstore.read_program_costs().unwrap());
+    let mut cost_tracker = CostTracker::default();
+    let mut cost_tracker_stats = CostTrackerStats::default();
+
+    for entry in &entries {
+        transactions += entry.transactions.len();
+        for transaction in &entry.transactions {
+            programs += transaction.message().instructions.len();
+            let tx_cost = cost_model.calculate_cost(transaction, true);
+            if cost_tracker
+                .try_add(transaction, &tx_cost, &mut cost_tracker_stats)
+                .is_err()
+            {
+                println!(
+                    "Slot: {}, CostModel rejected transaction {:?}, stats {:?}!",
+                    slot,
+                    transaction,
+                    cost_tracker.get_stats()
+                );
+            }
+            for instruction in &transaction.message().instructions {
+                let program_id =
+                    transaction.message().account_keys[instruction.program_id_index as usize];
+                *program_ids.entry(program_id).or_insert(0) += 1;
+            }
+        }
+    }
+
+    println!(
+        "Slot: {}, Entries: {}, Transactions: {}, Programs {}, {:?}",
+        slot,
+        entries.len(),
+        transactions,
+        programs,
+        cost_tracker.get_stats()
+    );
+    println!("  Programs: {:?}", program_ids);
+
+    Ok(())
+}
+
 fn open_genesis_config_by(ledger_path: &Path, matches: &ArgMatches<'_>) -> GenesisConfig {
     let max_genesis_archive_unpacked_size =
         value_t_or_exit!(matches, "max_genesis_archive_unpacked_size", u64);
@@ -1414,6 +1472,20 @@ fn main() {
                 .about("Output statistics in JSON format about \
                         all column families in the ledger rocksdb")
         )
+        .subcommand(
+            SubCommand::with_name("compute-slot-cost")
+            .about("runs cost_model over the block at the given slots, \
+                   computes how expensive a block was based on cost_model")
+            .arg(
+                Arg::with_name("slots")
+                    .index(1)
+                    .value_name("SLOTS")
+                    .validator(is_slot)
+                    .multiple(true)
+                    .takes_value(true)
+                    .help("Slots that their blocks are computed for cost, default to all slots in ledger"),
+            )
+        )
         .get_matches();
 
     info!("{} {}", crate_name!(), solana_version::version!());
@@ -2964,6 +3036,28 @@ fn main() {
             ));
             println!("Ok.");
         }
+        ("compute-slot-cost", Some(arg_matches)) => {
+            let blockstore = open_blockstore(
+                &ledger_path,
+                AccessType::TryPrimaryThenSecondary,
+                wal_recovery_mode,
+            );
+
+            let mut slots: Vec<u64> = vec![];
+            if !arg_matches.is_present("slots") {
+                if let Ok(metas) = blockstore.slot_meta_iterator(0) {
+                    slots = metas.map(|(slot, _)| slot).collect();
+                }
+            } else {
+                slots = values_t_or_exit!(arg_matches, "slots", Slot);
+            }
+
+            for slot in slots {
+                if let Err(err) = compute_slot_cost(&blockstore, slot) {
+                    eprintln!("{}", err);
+                }
+            }
+        }
         ("", _) => {
             eprintln!("{}", matches.usage());
             exit(1);

ledger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-ledger"
-version = "1.7.15"
+version = "1.8.1"
 description = "Solana ledger"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -33,21 +33,21 @@ rayon = "1.5.0"
 serde = "1.0.122"
 serde_bytes = "0.11.5"
 sha2 = "0.9.2"
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.7.15" }
-solana-frozen-abi = { path = "../frozen-abi", version = "=1.7.15" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.7.15" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.15" }
-solana-logger = { path = "../logger", version = "=1.7.15" }
-solana-measure = { path = "../measure", version = "=1.7.15" }
-solana-merkle-tree = { path = "../merkle-tree", version = "=1.7.15" }
-solana-metrics = { path = "../metrics", version = "=1.7.15" }
-solana-perf = { path = "../perf", version = "=1.7.15" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.7.15" }
-solana-runtime = { path = "../runtime", version = "=1.7.15" }
-solana-sdk = { path = "../sdk", version = "=1.7.15" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.7.15" }
-solana-storage-proto = { path = "../storage-proto", version = "=1.7.15" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.15" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.8.1" }
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.8.1" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.8.1" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.8.1" }
+solana-logger = { path = "../logger", version = "=1.8.1" }
+solana-measure = { path = "../measure", version = "=1.8.1" }
+solana-merkle-tree = { path = "../merkle-tree", version = "=1.8.1" }
+solana-metrics = { path = "../metrics", version = "=1.8.1" }
+solana-perf = { path = "../perf", version = "=1.8.1" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.8.1" }
+solana-runtime = { path = "../runtime", version = "=1.8.1" }
+solana-sdk = { path = "../sdk", version = "=1.8.1" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "=1.8.1" }
+solana-storage-proto = { path = "../storage-proto", version = "=1.8.1" }
+solana-vote-program = { path = "../programs/vote", version = "=1.8.1" }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
@@ -72,7 +72,7 @@ features = ["lz4"]
 [dev-dependencies]
 assert_matches = "1.3.0"
 matches = "0.1.6"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.15" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.8.1" }
 
 [build-dependencies]
 rustc_version = "0.2"

ledger/src/bank_forks_utils.rs

@@ -9,6 +9,7 @@ use crate::{
 };
 use log::*;
 use solana_runtime::{
+    accounts_update_notifier_interface::AccountsUpdateNotifier,
     bank_forks::{ArchiveFormat, BankForks, SnapshotConfig},
     snapshot_utils,
 };
@@ -33,6 +34,7 @@ fn to_loadresult(
 ///
 /// If a snapshot config is given, and a snapshot is found, it will be loaded.  Otherwise, load
 /// from genesis.
+#[allow(clippy::too_many_arguments)]
 pub fn load(
     genesis_config: &GenesisConfig,
     blockstore: &Blockstore,
@@ -42,6 +44,7 @@ pub fn load(
     process_options: ProcessOptions,
     transaction_status_sender: Option<&TransactionStatusSender>,
     cache_block_meta_sender: Option<&CacheBlockMetaSender>,
+    accounts_update_notifier: Option<AccountsUpdateNotifier>,
 ) -> LoadResult {
     if let Some(snapshot_config) = snapshot_config.as_ref() {
         info!(
@@ -70,6 +73,7 @@ pub fn load(
                 archive_slot,
                 archive_hash,
                 archive_format,
+                accounts_update_notifier,
             );
         } else {
             info!("No snapshot package available; will load from genesis");
@@ -84,6 +88,7 @@ pub fn load(
         account_paths,
         process_options,
         cache_block_meta_sender,
+        accounts_update_notifier,
     )
 }
 
@@ -93,6 +98,7 @@ fn load_from_genesis(
     account_paths: Vec<PathBuf>,
     process_options: ProcessOptions,
     cache_block_meta_sender: Option<&CacheBlockMetaSender>,
+    accounts_update_notifier: Option<AccountsUpdateNotifier>,
 ) -> LoadResult {
     info!("Processing ledger from genesis");
     to_loadresult(
@@ -102,6 +108,7 @@ fn load_from_genesis(
             account_paths,
             process_options,
             cache_block_meta_sender,
+            accounts_update_notifier,
         ),
         None,
     )
@@ -121,6 +128,7 @@ fn load_from_snapshot(
     archive_slot: Slot,
     archive_hash: Hash,
     archive_format: ArchiveFormat,
+    accounts_update_notifier: Option<AccountsUpdateNotifier>,
 ) -> LoadResult {
     info!("Loading snapshot package: {:?}", archive_filename);
 
@@ -145,6 +153,7 @@ fn load_from_snapshot(
         process_options.shrink_ratio,
         process_options.accounts_db_test_hash_calculation,
         process_options.accounts_db_skip_shrink,
+        accounts_update_notifier,
     )
     .expect("Load from snapshot failed");
     if let Some(shrink_paths) = shrink_paths {