chore: bump async from 2.6.3 to 2.6.4 in /docs

Bumps [async](https://github.com/caolan/async) from 2.6.3 to 2.6.4.
- [Release notes](https://github.com/caolan/async/releases)
- [Changelog](https://github.com/caolan/async/blob/v2.6.4/CHANGELOG.md)
- [Commits](https://github.com/caolan/async/compare/v2.6.3...v2.6.4)

---
updated-dependencies:
- dependency-name: async
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/ISSUE_TEMPLATE.md

@@ -1,6 +0,0 @@
-#### Problem
-
-
-
-#### Proposed Solution
-

.github/ISSUE_TEMPLATE/0-general.md

@@ -0,0 +1,12 @@
+---
+name: General Issue
+about: Create a report describing a problem and a proposed solution
+title: ''
+assignees: ''
+---
+
+#### Problem
+
+
+
+#### Proposed Solution

.github/ISSUE_TEMPLATE/1-feature-gate.yml

@@ -0,0 +1,70 @@
+name: Feature Gate Tracker
+description: Track the development and status of an on-chain feature
+title: "Feature Gate: "
+labels: ["feature-gate"]
+body:
+  - type: markdown
+    attributes:
+      value: >
+        Steps to add a new feature are outlined below. Note that these steps only cover
+        the process of getting a feature into the core Solana code.
+
+        - For features that are unambiguously good (ie bug fixes), these steps are sufficient.
+
+        - For features that should go up for community vote (ie fee structure changes), more
+        information on the additional steps to follow can be found at:
+        <https://spl.solana.com/feature-proposal#feature-proposal-life-cycle>
+
+        1. Generate a new keypair with `solana-keygen new --outfile feature.json --no-passphrase`
+            - Keypairs should be held by core contributors only. If you're a non-core contirbutor going
+            through these steps, the PR process will facilitate a keypair holder being picked. That
+            person will generate the keypair, provide pubkey for PR, and ultimately enable the feature.
+
+        2. Add a public module for the feature, specifying keypair pubkey as the id with
+        `solana_sdk::declare_id!()` within the module. Additionally, add an entry to `FEATURE_NAMES` map.
+
+        3. Add desired logic to check for and switch on feature availability.
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      placeholder: Describe why the new feature gate is needed and any necessary conditions for its activation
+    validations:
+      required: true
+  - type: input
+    id: id
+    attributes:
+      label: Feature ID
+      description: The public key of the feature account
+    validations:
+      required: true
+  - type: dropdown
+    id: activation-method
+    attributes:
+      label: Activation Method
+      options:
+        - Single Core Contributor
+        - Staked Validator Vote
+    validations:
+      required: true
+  - type: input
+    id: testnet
+    attributes:
+      label: Testnet Activation Epoch
+      placeholder: Edit this response when feature is activated on this cluster
+    validations:
+      required: false
+  - type: input
+    id: devnet
+    attributes:
+      label: Devnet Activation Epoch
+      placeholder: Edit this response when feature is activated on this cluster
+    validations:
+      required: false
+  - type: input
+    id: mainnet-beta
+    attributes:
+      label: Mainnet-Beta Activation Epoch
+      placeholder: Edit this response when feature is activated on this cluster
+    validations:
+      required: false

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,9 +1,9 @@
 #### Problem
 
 
-
 #### Summary of Changes
 
 
-
 Fixes #
+<!-- OPTIONAL: Feature Gate Issue: # -->
+<!-- Don't forget to add the "feature-gate" label -->

.github/workflows/autolock_bot_PR.txt

@@ -0,0 +1,37 @@
+name: 'Autolock RitBot for for PR'
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+permissions:
+  issues: write
+  pull-requests: write
+
+concurrency:
+  group: lock
+
+jobs:
+  action:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: dessant/lock-threads@v3
+        with:
+          
+          github-token: ${{ github.token }}
+          pr-inactive-days: '14'
+          exclude-pr-created-before: ''
+          exclude-pr-created-after: ''
+          exclude-pr-created-between: ''
+          exclude-pr-closed-before: ''
+          exclude-pr-closed-after: ''
+          exclude-pr-closed-between: ''
+          include-any-pr-labels: 'automerge'
+          include-all-pr-labels: ''
+          exclude-any-pr-labels: ''
+          add-pr-labels: 'locked PR'
+          remove-pr-labels: ''
+          pr-comment: 'This PR has been automatically locked since there has not been any activity in past 14 days after it was merged.'
+          pr-lock-reason: 'resolved'
+          log-output: true

.github/workflows/autolock_bot_closed_issue.txt

@@ -0,0 +1,38 @@
+name: 'Autolock NaviBot for closed issue'
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+permissions:
+  issues: write
+  pull-requests: write
+
+concurrency:
+  group: lock
+
+jobs:
+  action:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: dessant/lock-threads@v3
+        with:
+          
+          github-token: ${{ github.token }}
+          issue-inactive-days: '7'
+          exclude-issue-created-before: ''
+          exclude-issue-created-after: ''
+          exclude-issue-created-between: ''
+          exclude-issue-closed-before: ''
+          exclude-issue-closed-after: ''
+          exclude-issue-closed-between: ''
+          include-any-issue-labels: ''
+          include-all-issue-labels: ''
+          exclude-any-issue-labels: ''
+          add-issue-labels: 'locked issue'
+          remove-issue-labels: ''
+          issue-comment: 'This issue has been automatically locked since there has not been any activity in past 7 days after it was closed. Please open a new issue for related bugs.'
+          issue-lock-reason: 'resolved'
+          process-only: 'issues'
+          log-output: true

.mergify.yml

@@ -24,6 +24,7 @@ pull_request_rules:
       - "#approved-reviews-by=0"
       - "#commented-reviews-by=0"
       - "#changes-requested-reviews-by=0"
+      - "#review-requested=0"
     actions:
       request_reviews:
         teams:
@@ -34,6 +35,7 @@ pull_request_rules:
         - status-success=buildkite/solana
         - status-success=ci-gate
         - label=automerge
+        - label!=no-automerge
         - author≠@dont-squash-my-commits
         - or:
           # only require travis success if docs files changed
@@ -60,6 +62,7 @@ pull_request_rules:
         - status-success=Travis CI - Pull Request
         - status-success=ci-gate
         - label=automerge
+        - label!=no-automerge
         - author=@dont-squash-my-commits
         - or:
           # only require explorer checks if explorer files changed
@@ -93,26 +96,52 @@ pull_request_rules:
       - author=mergify[bot]
       - head~=^mergify/bp/
       - "#status-failure=0"
+      - "-merged"
+      - label!=no-automerge
     actions:
       label:
         add:
           - automerge
-  - name: v1.8 backport
+  - name: v1.9 feature-gate backport
     conditions:
-      - label=v1.8
+      - label=v1.9
+      - label=feature-gate
     actions:
       backport:
         ignore_conflicts: true
+        labels:
+          - feature-gate
         branches:
-          - v1.8
-  - name: v1.9 backport
+          - v1.9
+  - name: v1.9 non-feature-gate backport
     conditions:
       - label=v1.9
+      - label!=feature-gate
     actions:
       backport:
         ignore_conflicts: true
         branches:
           - v1.9
+  - name: v1.10 feature-gate backport
+    conditions:
+      - label=v1.10
+      - label=feature-gate
+    actions:
+      backport:
+        ignore_conflicts: true
+        labels:
+          - feature-gate
+        branches:
+          - v1.10
+  - name: v1.10 non-feature-gate backport
+    conditions:
+      - label=v1.10
+      - label!=feature-gate
+    actions:
+      backport:
+        ignore_conflicts: true
+        branches:
+          - v1.10
 
 commands_restrictions:
   # The author of copied PRs is the Mergify user.

CONTRIBUTING.md

@@ -146,6 +146,31 @@ the subject lines of the git commits contained in the PR. It's especially
 generous (and not expected) to rebase or reword commits such that each change
 matches the logical flow in your PR description.
 
+### The PR / Issue Labels
+Labels make it easier to manage and track PRs / issues.  Below some common labels
+that we use in Solana.  For the complete list of labels, please refer to the
+[label page](https://github.com/solana-labs/solana/issues/labels):
+
+* "feature-gate": when you add a new feature gate or modify the behavior of
+an existing feature gate, please add the "feature-gate" label to your PR.
+New feature gates should also always have a corresponding tracking issue
+(go to "New Issue" -> "Feature Gate Tracker [Get Started](https://github.com/solana-labs/solana/issues/new?assignees=&labels=feature-gate&template=1-feature-gate.yml&title=Feature+Gate%3A+)")
+and should be updated each time the feature is activated on a cluster.
+
+* "automerge": When a PR is labelled with "automerge", the PR will be
+automically merged once CI passes.  In general, this label should only
+be used for small hot-fix (fewer than 100 lines) or automatic generated
+PRs.  If you're uncertain, it's usually the case that the PR is not
+qualified as "automerge".
+
+* "good first issue": If you happen to find an issue that is non-urgent and
+self-contained with moderate scope, you might want to consider attaching
+"good first issue" to it as it might be a good practice for newcomers.
+
+* "rust": this pull request updates Rust code.
+
+* "javascript": this pull request updates Javascript code.
+
 ### When will my PR be reviewed?
 
 PRs are typically reviewed and merged in under 7 days. If your PR has been open

Cargo.toml

@@ -59,8 +59,6 @@ members = [
     "rayon-threadlimit",
     "rbpf-cli",
     "remote-wallet",
-    "replica-lib",
-    "replica-node",
     "rpc",
     "rpc-test",
     "runtime",
@@ -88,3 +86,6 @@ members = [
 exclude = [
     "programs/bpf",
 ]
+
+# This prevents a Travis CI error when building for Windows.
+resolver = "2"

README.md

@@ -35,7 +35,7 @@ On Linux systems you may need to install libssl-dev, pkg-config, zlib1g-dev, etc
 
 ```bash
 $ sudo apt-get update
-$ sudo apt-get install libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang make
+$ sudo apt-get install libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang cmake make
 ```
 
 ## **2. Download the source code.**

SECURITY.md

@@ -11,7 +11,7 @@
 email to security@solana.com and provide your github username so we can add you
 to a new draft security advisory for further discussion.
 
-Expect a response as fast as possible, within one business day at the latest.
+Expect a response as fast as possible, typically within 72 hours.
 
 <a name="bounty"></a>
 ## Security Bug Bounties

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.10.3"
+version = "1.11.0"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,9 +19,9 @@ lazy_static = "1.4.0"
 serde = "1.0.136"
 serde_derive = "1.0.103"
 serde_json = "1.0.79"
-solana-config-program = { path = "../programs/config", version = "=1.10.3" }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
-solana-vote-program = { path = "../programs/vote", version = "=1.10.3" }
+solana-config-program = { path = "../programs/config", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
+solana-vote-program = { path = "../programs/vote", version = "=1.11.0" }
 spl-token = { version = "=3.2.0", features = ["no-entrypoint"] }
 thiserror = "1.0"
 zstd = "0.11.1"

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-accounts-bench"
-version = "1.10.3"
+version = "1.11.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -12,11 +12,11 @@ publish = false
 clap = "2.33.1"
 log = "0.4.14"
 rayon = "1.5.1"
-solana-logger = { path = "../logger", version = "=1.10.3" }
-solana-measure = { path = "../measure", version = "=1.10.3" }
-solana-runtime = { path = "../runtime", version = "=1.10.3" }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
-solana-version = { path = "../version", version = "=1.10.3" }
+solana-logger = { path = "../logger", version = "=1.11.0" }
+solana-measure = { path = "../measure", version = "=1.11.0" }
+solana-runtime = { path = "../runtime", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
+solana-version = { path = "../version", version = "=1.11.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accounts-bench/src/main.rs

@@ -6,12 +6,18 @@ use {
     rayon::prelude::*,
     solana_measure::measure::Measure,
     solana_runtime::{
-        accounts::{create_test_accounts, update_accounts_bench, Accounts},
+        accounts::{
+            test_utils::{create_test_accounts, update_accounts_bench},
+            Accounts,
+        },
         accounts_db::AccountShrinkThreshold,
         accounts_index::AccountSecondaryIndexes,
         ancestors::Ancestors,
+        rent_collector::RentCollector,
+    },
+    solana_sdk::{
+        genesis_config::ClusterType, pubkey::Pubkey, sysvar::epoch_schedule::EpochSchedule,
     },
-    solana_sdk::{genesis_config::ClusterType, pubkey::Pubkey},
     std::{env, fs, path::PathBuf},
 };
 
@@ -114,7 +120,12 @@ fn main() {
         } else {
             let mut pubkeys: Vec<Pubkey> = vec![];
             let mut time = Measure::start("hash");
-            let results = accounts.accounts_db.update_accounts_hash(0, &ancestors);
+            let results = accounts.accounts_db.update_accounts_hash(
+                0,
+                &ancestors,
+                &EpochSchedule::default(),
+                &RentCollector::default(),
+            );
             time.stop();
             let mut time_store = Measure::start("hash using store");
             let results_store = accounts.accounts_db.update_accounts_hash_with_index_option(
@@ -124,7 +135,8 @@ fn main() {
                 &ancestors,
                 None,
                 false,
-                None,
+                &EpochSchedule::default(),
+                &RentCollector::default(),
                 false,
             );
             time_store.stop();

accounts-cluster-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-accounts-cluster-bench"
-version = "1.10.3"
+version = "1.11.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,25 +13,25 @@ clap = "2.33.1"
 log = "0.4.14"
 rand = "0.7.0"
 rayon = "1.5.1"
-solana-account-decoder = { path = "../account-decoder", version = "=1.10.3" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.10.3" }
-solana-client = { path = "../client", version = "=1.10.3" }
-solana-faucet = { path = "../faucet", version = "=1.10.3" }
-solana-gossip = { path = "../gossip", version = "=1.10.3" }
-solana-logger = { path = "../logger", version = "=1.10.3" }
-solana-measure = { path = "../measure", version = "=1.10.3" }
-solana-net-utils = { path = "../net-utils", version = "=1.10.3" }
-solana-runtime = { path = "../runtime", version = "=1.10.3" }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
-solana-streamer = { path = "../streamer", version = "=1.10.3" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.3" }
-solana-version = { path = "../version", version = "=1.10.3" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.11.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.11.0" }
+solana-client = { path = "../client", version = "=1.11.0" }
+solana-faucet = { path = "../faucet", version = "=1.11.0" }
+solana-gossip = { path = "../gossip", version = "=1.11.0" }
+solana-logger = { path = "../logger", version = "=1.11.0" }
+solana-measure = { path = "../measure", version = "=1.11.0" }
+solana-net-utils = { path = "../net-utils", version = "=1.11.0" }
+solana-runtime = { path = "../runtime", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
+solana-streamer = { path = "../streamer", version = "=1.11.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.11.0" }
+solana-version = { path = "../version", version = "=1.11.0" }
 spl-token = { version = "=3.2.0", features = ["no-entrypoint"] }
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "=1.10.3" }
-solana-local-cluster = { path = "../local-cluster", version = "=1.10.3" }
-solana-test-validator = { path = "../test-validator", version = "=1.10.3" }
+solana-core = { path = "../core", version = "=1.11.0" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.11.0" }
+solana-test-validator = { path = "../test-validator", version = "=1.11.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-banking-bench"
-version = "1.10.3"
+version = "1.11.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,17 +14,17 @@ crossbeam-channel = "0.5"
 log = "0.4.14"
 rand = "0.7.0"
 rayon = "1.5.1"
-solana-core = { path = "../core", version = "=1.10.3" }
-solana-gossip = { path = "../gossip", version = "=1.10.3" }
-solana-ledger = { path = "../ledger", version = "=1.10.3" }
-solana-logger = { path = "../logger", version = "=1.10.3" }
-solana-measure = { path = "../measure", version = "=1.10.3" }
-solana-perf = { path = "../perf", version = "=1.10.3" }
-solana-poh = { path = "../poh", version = "=1.10.3" }
-solana-runtime = { path = "../runtime", version = "=1.10.3" }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
-solana-streamer = { path = "../streamer", version = "=1.10.3" }
-solana-version = { path = "../version", version = "=1.10.3" }
+solana-core = { path = "../core", version = "=1.11.0" }
+solana-gossip = { path = "../gossip", version = "=1.11.0" }
+solana-ledger = { path = "../ledger", version = "=1.11.0" }
+solana-logger = { path = "../logger", version = "=1.11.0" }
+solana-measure = { path = "../measure", version = "=1.11.0" }
+solana-perf = { path = "../perf", version = "=1.11.0" }
+solana-poh = { path = "../poh", version = "=1.11.0" }
+solana-runtime = { path = "../runtime", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
+solana-streamer = { path = "../streamer", version = "=1.11.0" }
+solana-version = { path = "../version", version = "=1.11.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banks-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-client"
-version = "1.10.3"
+version = "1.11.0"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,17 +12,17 @@ edition = "2021"
 [dependencies]
 borsh = "0.9.3"
 futures = "0.3"
-solana-banks-interface = { path = "../banks-interface", version = "=1.10.3" }
-solana-program = { path = "../sdk/program", version = "=1.10.3" }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
+solana-banks-interface = { path = "../banks-interface", version = "=1.11.0" }
+solana-program = { path = "../sdk/program", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
 tarpc = { version = "0.27.2", features = ["full"] }
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 
 [dev-dependencies]
-solana-banks-server = { path = "../banks-server", version = "=1.10.3" }
-solana-runtime = { path = "../runtime", version = "=1.10.3" }
+solana-banks-server = { path = "../banks-server", version = "=1.11.0" }
+solana-runtime = { path = "../runtime", version = "=1.11.0" }
 
 [lib]
 crate-type = ["lib"]

banks-client/src/error.rs

@@ -1,5 +1,8 @@
 use {
-    solana_sdk::{transaction::TransactionError, transport::TransportError},
+    solana_sdk::{
+        transaction::TransactionError, transaction_context::TransactionReturnData,
+        transport::TransportError,
+    },
     std::io,
     tarpc::client::RpcError,
     thiserror::Error,
@@ -25,6 +28,7 @@ pub enum BanksClientError {
         err: TransactionError,
         logs: Vec<String>,
         units_consumed: u64,
+        return_data: Option<TransactionReturnData>,
     },
 }
 

banks-client/src/lib.rs

@@ -247,6 +247,7 @@ impl BanksClient {
                 err,
                 logs: simulation_details.logs,
                 units_consumed: simulation_details.units_consumed,
+                return_data: simulation_details.return_data,
             }),
             BanksTransactionResultWithSimulation {
                 result: Some(result),

banks-interface/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-interface"
-version = "1.10.3"
+version = "1.11.0"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,7 +11,7 @@ edition = "2021"
 
 [dependencies]
 serde = { version = "1.0.136", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
 tarpc = { version = "0.27.2", features = ["full"] }
 
 [lib]

banks-interface/src/lib.rs

@@ -12,6 +12,7 @@ use {
         pubkey::Pubkey,
         signature::Signature,
         transaction::{self, Transaction, TransactionError},
+        transaction_context::TransactionReturnData,
     },
 };
 
@@ -35,6 +36,7 @@ pub struct TransactionStatus {
 pub struct TransactionSimulationDetails {
     pub logs: Vec<String>,
     pub units_consumed: u64,
+    pub return_data: Option<TransactionReturnData>,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]

banks-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-server"
-version = "1.10.3"
+version = "1.11.0"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,10 +13,10 @@ edition = "2021"
 bincode = "1.3.3"
 crossbeam-channel = "0.5"
 futures = "0.3"
-solana-banks-interface = { path = "../banks-interface", version = "=1.10.3" }
-solana-runtime = { path = "../runtime", version = "=1.10.3" }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
-solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.10.3" }
+solana-banks-interface = { path = "../banks-interface", version = "=1.11.0" }
+solana-runtime = { path = "../runtime", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
+solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.11.0" }
 tarpc = { version = "0.27.2", features = ["full"] }
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }

banks-server/src/banks_server.rs

@@ -24,7 +24,7 @@ use {
         transaction::{self, SanitizedTransaction, Transaction},
     },
     solana_send_transaction_service::{
-        send_transaction_service::{SendTransactionService, TransactionInfo},
+        send_transaction_service::{SendTransactionService, TransactionInfo, DEFAULT_TPU_USE_QUIC},
         tpu_info::NullTpuInfo,
     },
     std::{
@@ -266,6 +266,7 @@ impl Banks for BanksServer {
             logs,
             post_simulation_accounts: _,
             units_consumed,
+            return_data,
         } = self
             .bank(commitment)
             .simulate_transaction_unchecked(sanitized_transaction)
@@ -275,6 +276,7 @@ impl Banks for BanksServer {
                 simulation_details: Some(TransactionSimulationDetails {
                     logs,
                     units_consumed,
+                    return_data,
                 }),
             };
         }
@@ -399,6 +401,7 @@ pub async fn start_tcp_server(
                 receiver,
                 5_000,
                 0,
+                DEFAULT_TPU_USE_QUIC,
             );
 
             let server = BanksServer::new(

bench-streamer/Cargo.toml

@@ -2,18 +2,18 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-bench-streamer"
-version = "1.10.3"
+version = "1.11.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 publish = false
 
 [dependencies]
-clap = "2.33.1"
 crossbeam-channel = "0.5"
-solana-net-utils = { path = "../net-utils", version = "=1.10.3" }
-solana-streamer = { path = "../streamer", version = "=1.10.3" }
-solana-version = { path = "../version", version = "=1.10.3" }
+clap = { version = "3.1.5", features = ["cargo"] }
+solana-net-utils = { path = "../net-utils", version = "=1.11.0" }
+solana-streamer = { path = "../streamer", version = "=1.11.0" }
+solana-version = { path = "../version", version = "=1.11.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/src/main.rs

@@ -1,6 +1,6 @@
 #![allow(clippy::integer_arithmetic)]
 use {
-    clap::{crate_description, crate_name, value_t, App, Arg},
+    clap::{crate_description, crate_name, Arg, Command},
     crossbeam_channel::unbounded,
     solana_streamer::{
         packet::{Packet, PacketBatch, PacketBatchRecycler, PACKET_DATA_SIZE},
@@ -57,18 +57,18 @@ fn sink(exit: Arc<AtomicBool>, rvs: Arc<AtomicUsize>, r: PacketBatchReceiver) ->
 fn main() -> Result<()> {
     let mut num_sockets = 1usize;
 
-    let matches = App::new(crate_name!())
+    let matches = Command::new(crate_name!())
         .about(crate_description!())
         .version(solana_version::version!())
         .arg(
-            Arg::with_name("num-recv-sockets")
+            Arg::new("num-recv-sockets")
                 .long("num-recv-sockets")
                 .value_name("NUM")
                 .takes_value(true)
                 .help("Use NUM receive sockets"),
         )
         .arg(
-            Arg::with_name("num-producers")
+            Arg::new("num-producers")
                 .long("num-producers")
                 .value_name("NUM")
                 .takes_value(true)
@@ -80,7 +80,7 @@ fn main() -> Result<()> {
         num_sockets = max(num_sockets, n.to_string().parse().expect("integer"));
     }
 
-    let num_producers = value_t!(matches, "num_producers", u64).unwrap_or(4);
+    let num_producers: u64 = matches.value_of_t("num_producers").unwrap_or(4);
 
     let port = 0;
     let ip_addr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0));

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-bench-tps"
-version = "1.10.3"
+version = "1.11.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,23 +15,28 @@ log = "0.4.14"
 rayon = "1.5.1"
 serde_json = "1.0.79"
 serde_yaml = "0.8.23"
-solana-client = { path = "../client", version = "=1.10.3" }
-solana-core = { path = "../core", version = "=1.10.3" }
-solana-faucet = { path = "../faucet", version = "=1.10.3" }
-solana-genesis = { path = "../genesis", version = "=1.10.3" }
-solana-gossip = { path = "../gossip", version = "=1.10.3" }
-solana-logger = { path = "../logger", version = "=1.10.3" }
-solana-measure = { path = "../measure", version = "=1.10.3" }
-solana-metrics = { path = "../metrics", version = "=1.10.3" }
-solana-net-utils = { path = "../net-utils", version = "=1.10.3" }
-solana-runtime = { path = "../runtime", version = "=1.10.3" }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
-solana-streamer = { path = "../streamer", version = "=1.10.3" }
-solana-version = { path = "../version", version = "=1.10.3" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.11.0" }
+solana-cli-config = { path = "../cli-config", version = "=1.11.0" }
+solana-client = { path = "../client", version = "=1.11.0" }
+solana-core = { path = "../core", version = "=1.11.0" }
+solana-faucet = { path = "../faucet", version = "=1.11.0" }
+solana-genesis = { path = "../genesis", version = "=1.11.0" }
+solana-gossip = { path = "../gossip", version = "=1.11.0" }
+solana-logger = { path = "../logger", version = "=1.11.0" }
+solana-measure = { path = "../measure", version = "=1.11.0" }
+solana-metrics = { path = "../metrics", version = "=1.11.0" }
+solana-net-utils = { path = "../net-utils", version = "=1.11.0" }
+solana-rpc = { path = "../rpc", version = "=1.11.0" }
+solana-runtime = { path = "../runtime", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
+solana-streamer = { path = "../streamer", version = "=1.11.0" }
+solana-version = { path = "../version", version = "=1.11.0" }
+thiserror = "1.0"
 
 [dev-dependencies]
 serial_test = "0.6.0"
-solana-local-cluster = { path = "../local-cluster", version = "=1.10.3" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.11.0" }
+solana-test-validator = { path = "../test-validator", version = "=1.11.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/src/bench.rs

@@ -1,19 +1,21 @@
 use {
-    crate::cli::Config,
+    crate::{
+        bench_tps_client::*,
+        cli::Config,
+        perf_utils::{sample_txs, SampleStats},
+    },
     log::*,
     rayon::prelude::*,
-    solana_client::perf_utils::{sample_txs, SampleStats},
     solana_core::gen_keys::GenKeys,
-    solana_faucet::faucet::request_airdrop_transaction,
     solana_measure::measure::Measure,
     solana_metrics::{self, datapoint_info},
     solana_sdk::{
-        client::Client,
         clock::{DEFAULT_MS_PER_SLOT, DEFAULT_S_PER_SLOT, MAX_PROCESSING_AGE},
         commitment_config::CommitmentConfig,
         hash::Hash,
         instruction::{AccountMeta, Instruction},
         message::Message,
+        native_token::Sol,
         pubkey::Pubkey,
         signature::{Keypair, Signer},
         system_instruction, system_transaction,
@@ -22,7 +24,6 @@ use {
     },
     std::{
         collections::{HashSet, VecDeque},
-        net::SocketAddr,
         process::exit,
         sync::{
             atomic::{AtomicBool, AtomicIsize, AtomicUsize, Ordering},
@@ -38,16 +39,9 @@ const MAX_TX_QUEUE_AGE: u64 = (MAX_PROCESSING_AGE as f64 * DEFAULT_S_PER_SLOT) a
 
 pub const MAX_SPENDS_PER_TX: u64 = 4;
 
-#[derive(Debug)]
-pub enum BenchTpsError {
-    AirdropFailure,
-}
-
-pub type Result<T> = std::result::Result<T, BenchTpsError>;
-
 pub type SharedTransactions = Arc<RwLock<VecDeque<Vec<(Transaction, u64)>>>>;
 
-fn get_latest_blockhash<T: Client>(client: &T) -> Hash {
+fn get_latest_blockhash<T: BenchTpsClient>(client: &T) -> Hash {
     loop {
         match client.get_latest_blockhash_with_commitment(CommitmentConfig::processed()) {
             Ok((blockhash, _)) => return blockhash,
@@ -61,7 +55,7 @@ fn get_latest_blockhash<T: Client>(client: &T) -> Hash {
 
 fn wait_for_target_slots_per_epoch<T>(target_slots_per_epoch: u64, client: &Arc<T>)
 where
-    T: 'static + Client + Send + Sync,
+    T: 'static + BenchTpsClient + Send + Sync,
 {
     if target_slots_per_epoch != 0 {
         info!(
@@ -91,7 +85,7 @@ fn create_sampler_thread<T>(
     maxes: &Arc<RwLock<Vec<(String, SampleStats)>>>,
 ) -> JoinHandle<()>
 where
-    T: 'static + Client + Send + Sync,
+    T: 'static + BenchTpsClient + Send + Sync,
 {
     info!("Sampling TPS every {} second...", sample_period);
     let exit_signal = exit_signal.clone();
@@ -169,7 +163,7 @@ fn create_sender_threads<T>(
     shared_tx_active_thread_count: &Arc<AtomicIsize>,
 ) -> Vec<JoinHandle<()>>
 where
-    T: 'static + Client + Send + Sync,
+    T: 'static + BenchTpsClient + Send + Sync,
 {
     (0..threads)
         .map(|_| {
@@ -197,7 +191,7 @@ where
 
 pub fn do_bench_tps<T>(client: Arc<T>, config: Config, gen_keypairs: Vec<Keypair>) -> u64
 where
-    T: 'static + Client + Send + Sync,
+    T: 'static + BenchTpsClient + Send + Sync,
 {
     let Config {
         id,
@@ -391,7 +385,7 @@ fn generate_txs(
     }
 }
 
-fn get_new_latest_blockhash<T: Client>(client: &Arc<T>, blockhash: &Hash) -> Option<Hash> {
+fn get_new_latest_blockhash<T: BenchTpsClient>(client: &Arc<T>, blockhash: &Hash) -> Option<Hash> {
     let start = Instant::now();
     while start.elapsed().as_secs() < 5 {
         if let Ok(new_blockhash) = client.get_latest_blockhash() {
@@ -407,7 +401,7 @@ fn get_new_latest_blockhash<T: Client>(client: &Arc<T>, blockhash: &Hash) -> Opt
     None
 }
 
-fn poll_blockhash<T: Client>(
+fn poll_blockhash<T: BenchTpsClient>(
     exit_signal: &Arc<AtomicBool>,
     blockhash: &Arc<RwLock<Hash>>,
     client: &Arc<T>,
@@ -449,7 +443,7 @@ fn poll_blockhash<T: Client>(
     }
 }
 
-fn do_tx_transfers<T: Client>(
+fn do_tx_transfers<T: BenchTpsClient>(
     exit_signal: &Arc<AtomicBool>,
     shared_txs: &SharedTransactions,
     shared_tx_thread_count: &Arc<AtomicIsize>,
@@ -467,11 +461,7 @@ fn do_tx_transfers<T: Client>(
         };
         if let Some(txs0) = txs {
             shared_tx_thread_count.fetch_add(1, Ordering::Relaxed);
-            info!(
-                "Transferring 1 unit {} times... to {}",
-                txs0.len(),
-                client.as_ref().tpu_addr(),
-            );
+            info!("Transferring 1 unit {} times...", txs0.len());
             let tx_len = txs0.len();
             let transfer_start = Instant::now();
             let mut old_transactions = false;
@@ -487,7 +477,7 @@ fn do_tx_transfers<T: Client>(
                 transactions.push(tx.0);
             }
 
-            if let Err(error) = client.async_send_batch(transactions) {
+            if let Err(error) = client.send_batch(transactions) {
                 warn!("send_batch_sync in do_tx_transfers failed: {}", error);
             }
 
@@ -514,7 +504,11 @@ fn do_tx_transfers<T: Client>(
     }
 }
 
-fn verify_funding_transfer<T: Client>(client: &Arc<T>, tx: &Transaction, amount: u64) -> bool {
+fn verify_funding_transfer<T: BenchTpsClient>(
+    client: &Arc<T>,
+    tx: &Transaction,
+    amount: u64,
+) -> bool {
     for a in &tx.message().account_keys[1..] {
         match client.get_balance_with_commitment(a, CommitmentConfig::processed()) {
             Ok(balance) => return balance >= amount,
@@ -525,7 +519,7 @@ fn verify_funding_transfer<T: Client>(client: &Arc<T>, tx: &Transaction, amount:
 }
 
 trait FundingTransactions<'a> {
-    fn fund<T: 'static + Client + Send + Sync>(
+    fn fund<T: 'static + BenchTpsClient + Send + Sync>(
         &mut self,
         client: &Arc<T>,
         to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)],
@@ -533,12 +527,16 @@ trait FundingTransactions<'a> {
     );
     fn make(&mut self, to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)]);
     fn sign(&mut self, blockhash: Hash);
-    fn send<T: Client>(&self, client: &Arc<T>);
-    fn verify<T: 'static + Client + Send + Sync>(&mut self, client: &Arc<T>, to_lamports: u64);
+    fn send<T: BenchTpsClient>(&self, client: &Arc<T>);
+    fn verify<T: 'static + BenchTpsClient + Send + Sync>(
+        &mut self,
+        client: &Arc<T>,
+        to_lamports: u64,
+    );
 }
 
 impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
-    fn fund<T: 'static + Client + Send + Sync>(
+    fn fund<T: 'static + BenchTpsClient + Send + Sync>(
         &mut self,
         client: &Arc<T>,
         to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)],
@@ -607,16 +605,20 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
         debug!("sign {} txs: {}us", self.len(), sign_txs.as_us());
     }
 
-    fn send<T: Client>(&self, client: &Arc<T>) {
+    fn send<T: BenchTpsClient>(&self, client: &Arc<T>) {
         let mut send_txs = Measure::start("send_txs");
         self.iter().for_each(|(_, tx)| {
-            client.async_send_transaction(tx.clone()).expect("transfer");
+            client.send_transaction(tx.clone()).expect("transfer");
         });
         send_txs.stop();
         debug!("send {} txs: {}us", self.len(), send_txs.as_us());
     }
 
-    fn verify<T: 'static + Client + Send + Sync>(&mut self, client: &Arc<T>, to_lamports: u64) {
+    fn verify<T: 'static + BenchTpsClient + Send + Sync>(
+        &mut self,
+        client: &Arc<T>,
+        to_lamports: u64,
+    ) {
         let starting_txs = self.len();
         let verified_txs = Arc::new(AtomicUsize::new(0));
         let too_many_failures = Arc::new(AtomicBool::new(false));
@@ -691,7 +693,7 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
 /// fund the dests keys by spending all of the source keys into MAX_SPENDS_PER_TX
 /// on every iteration.  This allows us to replay the transfers because the source is either empty,
 /// or full
-pub fn fund_keys<T: 'static + Client + Send + Sync>(
+pub fn fund_keys<T: 'static + BenchTpsClient + Send + Sync>(
     client: Arc<T>,
     source: &Keypair,
     dests: &[Keypair],
@@ -733,75 +735,6 @@ pub fn fund_keys<T: 'static + Client + Send + Sync>(
     }
 }
 
-pub fn airdrop_lamports<T: Client>(
-    client: &T,
-    faucet_addr: &SocketAddr,
-    id: &Keypair,
-    desired_balance: u64,
-) -> Result<()> {
-    let starting_balance = client.get_balance(&id.pubkey()).unwrap_or(0);
-    metrics_submit_lamport_balance(starting_balance);
-    info!("starting balance {}", starting_balance);
-
-    if starting_balance < desired_balance {
-        let airdrop_amount = desired_balance - starting_balance;
-        info!(
-            "Airdropping {:?} lamports from {} for {}",
-            airdrop_amount,
-            faucet_addr,
-            id.pubkey(),
-        );
-
-        let blockhash = get_latest_blockhash(client);
-        match request_airdrop_transaction(faucet_addr, &id.pubkey(), airdrop_amount, blockhash) {
-            Ok(transaction) => {
-                let mut tries = 0;
-                loop {
-                    tries += 1;
-                    let signature = client.async_send_transaction(transaction.clone()).unwrap();
-                    let result = client.poll_for_signature_confirmation(&signature, 1);
-
-                    if result.is_ok() {
-                        break;
-                    }
-                    if tries >= 5 {
-                        panic!(
-                            "Error requesting airdrop: to addr: {:?} amount: {} {:?}",
-                            faucet_addr, airdrop_amount, result
-                        )
-                    }
-                }
-            }
-            Err(err) => {
-                panic!(
-                    "Error requesting airdrop: {:?} to addr: {:?} amount: {}",
-                    err, faucet_addr, airdrop_amount
-                );
-            }
-        };
-
-        let current_balance = client
-            .get_balance_with_commitment(&id.pubkey(), CommitmentConfig::processed())
-            .unwrap_or_else(|e| {
-                info!("airdrop error {}", e);
-                starting_balance
-            });
-        info!("current balance {}...", current_balance);
-
-        metrics_submit_lamport_balance(current_balance);
-        if current_balance - starting_balance != airdrop_amount {
-            info!(
-                "Airdrop failed! {} {} {}",
-                id.pubkey(),
-                current_balance,
-                starting_balance
-            );
-            return Err(BenchTpsError::AirdropFailure);
-        }
-    }
-    Ok(())
-}
-
 fn compute_and_report_stats(
     maxes: &Arc<RwLock<Vec<(String, SampleStats)>>>,
     sample_period: u64,
@@ -885,15 +818,33 @@ pub fn generate_keypairs(seed_keypair: &Keypair, count: u64) -> (Vec<Keypair>, u
     (rnd.gen_n_keypairs(total_keys), extra)
 }
 
-pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
+pub fn generate_and_fund_keypairs<T: 'static + BenchTpsClient + Send + Sync>(
     client: Arc<T>,
-    faucet_addr: Option<SocketAddr>,
     funding_key: &Keypair,
     keypair_count: usize,
     lamports_per_account: u64,
 ) -> Result<Vec<Keypair>> {
+    let rent = client.get_minimum_balance_for_rent_exemption(0)?;
+    let lamports_per_account = lamports_per_account + rent;
+
     info!("Creating {} keypairs...", keypair_count);
     let (mut keypairs, extra) = generate_keypairs(funding_key, keypair_count as u64);
+    fund_keypairs(client, funding_key, &keypairs, extra, lamports_per_account)?;
+
+    // 'generate_keypairs' generates extra keys to be able to have size-aligned funding batches for fund_keys.
+    keypairs.truncate(keypair_count);
+
+    Ok(keypairs)
+}
+
+pub fn fund_keypairs<T: 'static + BenchTpsClient + Send + Sync>(
+    client: Arc<T>,
+    funding_key: &Keypair,
+    keypairs: &[Keypair],
+    extra: u64,
+    lamports_per_account: u64,
+) -> Result<()> {
+    let rent = client.get_minimum_balance_for_rent_exemption(0)?;
     info!("Get lamports...");
 
     // Sample the first keypair, to prevent lamport loss on repeated solana-bench-tps executions
@@ -901,7 +852,7 @@ pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
     let first_keypair_balance = client.get_balance(&first_key).unwrap_or(0);
 
     // Sample the last keypair, to check if funding was already completed
-    let last_key = keypairs[keypair_count - 1].pubkey();
+    let last_key = keypairs[keypairs.len() - 1].pubkey();
     let last_keypair_balance = client.get_balance(&last_key).unwrap_or(0);
 
     // Repeated runs will eat up keypair balances from transaction fees. In order to quickly
@@ -930,24 +881,35 @@ pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
             funding_key_balance, max_fee, lamports_per_account, extra, total
         );
 
-        if client.get_balance(&funding_key.pubkey()).unwrap_or(0) < total {
-            airdrop_lamports(client.as_ref(), &faucet_addr.unwrap(), funding_key, total)?;
+        if funding_key_balance < total + rent {
+            error!(
+                "funder has {}, needed {}",
+                Sol(funding_key_balance),
+                Sol(total)
+            );
+            let latest_blockhash = get_latest_blockhash(client.as_ref());
+            if client
+                .request_airdrop_with_blockhash(
+                    &funding_key.pubkey(),
+                    total + rent - funding_key_balance,
+                    &latest_blockhash,
+                )
+                .is_err()
+            {
+                return Err(BenchTpsError::AirdropFailure);
+            }
         }
 
         fund_keys(
             client,
             funding_key,
-            &keypairs,
+            keypairs,
             total,
             max_fee,
             lamports_per_account,
         );
     }
-
-    // 'generate_keypairs' generates extra keys to be able to have size-aligned funding batches for fund_keys.
-    keypairs.truncate(keypair_count);
-
-    Ok(keypairs)
+    Ok(())
 }
 
 #[cfg(test)]
@@ -956,14 +918,14 @@ mod tests {
         super::*,
         solana_runtime::{bank::Bank, bank_client::BankClient},
         solana_sdk::{
-            client::SyncClient, fee_calculator::FeeRateGovernor,
-            genesis_config::create_genesis_config,
+            fee_calculator::FeeRateGovernor, genesis_config::create_genesis_config,
+            native_token::sol_to_lamports,
         },
     };
 
     #[test]
     fn test_bench_tps_bank_client() {
-        let (genesis_config, id) = create_genesis_config(10_000);
+        let (genesis_config, id) = create_genesis_config(sol_to_lamports(10_000.0));
         let bank = Bank::new_for_tests(&genesis_config);
         let client = Arc::new(BankClient::new(bank));
 
@@ -976,48 +938,49 @@ mod tests {
 
         let keypair_count = config.tx_count * config.keypair_multiplier;
         let keypairs =
-            generate_and_fund_keypairs(client.clone(), None, &config.id, keypair_count, 20)
-                .unwrap();
+            generate_and_fund_keypairs(client.clone(), &config.id, keypair_count, 20).unwrap();
 
         do_bench_tps(client, config, keypairs);
     }
 
     #[test]
     fn test_bench_tps_fund_keys() {
-        let (genesis_config, id) = create_genesis_config(10_000);
+        let (genesis_config, id) = create_genesis_config(sol_to_lamports(10_000.0));
         let bank = Bank::new_for_tests(&genesis_config);
         let client = Arc::new(BankClient::new(bank));
         let keypair_count = 20;
         let lamports = 20;
+        let rent = client.get_minimum_balance_for_rent_exemption(0).unwrap();
 
         let keypairs =
-            generate_and_fund_keypairs(client.clone(), None, &id, keypair_count, lamports).unwrap();
+            generate_and_fund_keypairs(client.clone(), &id, keypair_count, lamports).unwrap();
 
         for kp in &keypairs {
             assert_eq!(
                 client
                     .get_balance_with_commitment(&kp.pubkey(), CommitmentConfig::processed())
                     .unwrap(),
-                lamports
+                lamports + rent
             );
         }
     }
 
     #[test]
     fn test_bench_tps_fund_keys_with_fees() {
-        let (mut genesis_config, id) = create_genesis_config(10_000);
+        let (mut genesis_config, id) = create_genesis_config(sol_to_lamports(10_000.0));
         let fee_rate_governor = FeeRateGovernor::new(11, 0);
         genesis_config.fee_rate_governor = fee_rate_governor;
         let bank = Bank::new_for_tests(&genesis_config);
         let client = Arc::new(BankClient::new(bank));
         let keypair_count = 20;
         let lamports = 20;
+        let rent = client.get_minimum_balance_for_rent_exemption(0).unwrap();
 
         let keypairs =
-            generate_and_fund_keypairs(client.clone(), None, &id, keypair_count, lamports).unwrap();
+            generate_and_fund_keypairs(client.clone(), &id, keypair_count, lamports).unwrap();
 
         for kp in &keypairs {
-            assert_eq!(client.get_balance(&kp.pubkey()).unwrap(), lamports);
+            assert_eq!(client.get_balance(&kp.pubkey()).unwrap(), lamports + rent);
         }
     }
 }

bench-tps/src/bench_tps_client.rs

@@ -0,0 +1,87 @@
+use {
+    solana_client::{client_error::ClientError, tpu_client::TpuSenderError},
+    solana_sdk::{
+        commitment_config::CommitmentConfig, epoch_info::EpochInfo, hash::Hash, message::Message,
+        pubkey::Pubkey, signature::Signature, transaction::Transaction, transport::TransportError,
+    },
+    thiserror::Error,
+};
+
+#[derive(Error, Debug)]
+pub enum BenchTpsError {
+    #[error("Airdrop failure")]
+    AirdropFailure,
+    #[error("IO error: {0:?}")]
+    IoError(#[from] std::io::Error),
+    #[error("Client error: {0:?}")]
+    ClientError(#[from] ClientError),
+    #[error("TpuClient error: {0:?}")]
+    TpuSenderError(#[from] TpuSenderError),
+    #[error("Transport error: {0:?}")]
+    TransportError(#[from] TransportError),
+    #[error("Custom error: {0}")]
+    Custom(String),
+}
+
+pub(crate) type Result<T> = std::result::Result<T, BenchTpsError>;
+
+pub trait BenchTpsClient {
+    /// Send a signed transaction without confirmation
+    fn send_transaction(&self, transaction: Transaction) -> Result<Signature>;
+
+    /// Send a batch of signed transactions without confirmation.
+    fn send_batch(&self, transactions: Vec<Transaction>) -> Result<()>;
+
+    /// Get latest blockhash
+    fn get_latest_blockhash(&self) -> Result<Hash>;
+
+    /// Get latest blockhash and its last valid block height, using explicit commitment
+    fn get_latest_blockhash_with_commitment(
+        &self,
+        commitment_config: CommitmentConfig,
+    ) -> Result<(Hash, u64)>;
+
+    /// Get transaction count
+    fn get_transaction_count(&self) -> Result<u64>;
+
+    /// Get transaction count, using explicit commitment
+    fn get_transaction_count_with_commitment(
+        &self,
+        commitment_config: CommitmentConfig,
+    ) -> Result<u64>;
+
+    /// Get epoch info
+    fn get_epoch_info(&self) -> Result<EpochInfo>;
+
+    /// Get account balance
+    fn get_balance(&self, pubkey: &Pubkey) -> Result<u64>;
+
+    /// Get account balance, using explicit commitment
+    fn get_balance_with_commitment(
+        &self,
+        pubkey: &Pubkey,
+        commitment_config: CommitmentConfig,
+    ) -> Result<u64>;
+
+    /// Calculate the fee for a `Message`
+    fn get_fee_for_message(&self, message: &Message) -> Result<u64>;
+
+    /// Get the rent-exempt minimum for an account
+    fn get_minimum_balance_for_rent_exemption(&self, data_len: usize) -> Result<u64>;
+
+    /// Return the address of client
+    fn addr(&self) -> String;
+
+    /// Request, submit, and confirm an airdrop transaction
+    fn request_airdrop_with_blockhash(
+        &self,
+        pubkey: &Pubkey,
+        lamports: u64,
+        recent_blockhash: &Hash,
+    ) -> Result<Signature>;
+}
+
+mod bank_client;
+mod rpc_client;
+mod thin_client;
+mod tpu_client;

bench-tps/src/bench_tps_client/bank_client.rs

@@ -0,0 +1,85 @@
+use {
+    crate::bench_tps_client::{BenchTpsClient, BenchTpsError, Result},
+    solana_runtime::bank_client::BankClient,
+    solana_sdk::{
+        client::{AsyncClient, SyncClient},
+        commitment_config::CommitmentConfig,
+        epoch_info::EpochInfo,
+        hash::Hash,
+        message::Message,
+        pubkey::Pubkey,
+        signature::Signature,
+        transaction::Transaction,
+    },
+};
+
+impl BenchTpsClient for BankClient {
+    fn send_transaction(&self, transaction: Transaction) -> Result<Signature> {
+        AsyncClient::async_send_transaction(self, transaction).map_err(|err| err.into())
+    }
+    fn send_batch(&self, transactions: Vec<Transaction>) -> Result<()> {
+        AsyncClient::async_send_batch(self, transactions).map_err(|err| err.into())
+    }
+    fn get_latest_blockhash(&self) -> Result<Hash> {
+        SyncClient::get_latest_blockhash(self).map_err(|err| err.into())
+    }
+
+    fn get_latest_blockhash_with_commitment(
+        &self,
+        commitment_config: CommitmentConfig,
+    ) -> Result<(Hash, u64)> {
+        SyncClient::get_latest_blockhash_with_commitment(self, commitment_config)
+            .map_err(|err| err.into())
+    }
+
+    fn get_transaction_count(&self) -> Result<u64> {
+        SyncClient::get_transaction_count(self).map_err(|err| err.into())
+    }
+
+    fn get_transaction_count_with_commitment(
+        &self,
+        commitment_config: CommitmentConfig,
+    ) -> Result<u64> {
+        SyncClient::get_transaction_count_with_commitment(self, commitment_config)
+            .map_err(|err| err.into())
+    }
+
+    fn get_epoch_info(&self) -> Result<EpochInfo> {
+        SyncClient::get_epoch_info(self).map_err(|err| err.into())
+    }
+
+    fn get_balance(&self, pubkey: &Pubkey) -> Result<u64> {
+        SyncClient::get_balance(self, pubkey).map_err(|err| err.into())
+    }
+
+    fn get_balance_with_commitment(
+        &self,
+        pubkey: &Pubkey,
+        commitment_config: CommitmentConfig,
+    ) -> Result<u64> {
+        SyncClient::get_balance_with_commitment(self, pubkey, commitment_config)
+            .map_err(|err| err.into())
+    }
+
+    fn get_fee_for_message(&self, message: &Message) -> Result<u64> {
+        SyncClient::get_fee_for_message(self, message).map_err(|err| err.into())
+    }
+
+    fn get_minimum_balance_for_rent_exemption(&self, data_len: usize) -> Result<u64> {
+        SyncClient::get_minimum_balance_for_rent_exemption(self, data_len).map_err(|err| err.into())
+    }
+
+    fn addr(&self) -> String {
+        "Local BankClient".to_string()
+    }
+
+    fn request_airdrop_with_blockhash(
+        &self,
+        _pubkey: &Pubkey,
+        _lamports: u64,
+        _recent_blockhash: &Hash,
+    ) -> Result<Signature> {
+        // BankClient doesn't support airdrops
+        Err(BenchTpsError::AirdropFailure)
+    }
+}

bench-tps/src/bench_tps_client/rpc_client.rs

@@ -0,0 +1,83 @@
+use {
+    crate::bench_tps_client::{BenchTpsClient, Result},
+    solana_client::rpc_client::RpcClient,
+    solana_sdk::{
+        commitment_config::CommitmentConfig, epoch_info::EpochInfo, hash::Hash, message::Message,
+        pubkey::Pubkey, signature::Signature, transaction::Transaction,
+    },
+};
+
+impl BenchTpsClient for RpcClient {
+    fn send_transaction(&self, transaction: Transaction) -> Result<Signature> {
+        RpcClient::send_transaction(self, &transaction).map_err(|err| err.into())
+    }
+    fn send_batch(&self, transactions: Vec<Transaction>) -> Result<()> {
+        for transaction in transactions {
+            BenchTpsClient::send_transaction(self, transaction)?;
+        }
+        Ok(())
+    }
+    fn get_latest_blockhash(&self) -> Result<Hash> {
+        RpcClient::get_latest_blockhash(self).map_err(|err| err.into())
+    }
+
+    fn get_latest_blockhash_with_commitment(
+        &self,
+        commitment_config: CommitmentConfig,
+    ) -> Result<(Hash, u64)> {
+        RpcClient::get_latest_blockhash_with_commitment(self, commitment_config)
+            .map_err(|err| err.into())
+    }
+
+    fn get_transaction_count(&self) -> Result<u64> {
+        RpcClient::get_transaction_count(self).map_err(|err| err.into())
+    }
+
+    fn get_transaction_count_with_commitment(
+        &self,
+        commitment_config: CommitmentConfig,
+    ) -> Result<u64> {
+        RpcClient::get_transaction_count_with_commitment(self, commitment_config)
+            .map_err(|err| err.into())
+    }
+
+    fn get_epoch_info(&self) -> Result<EpochInfo> {
+        RpcClient::get_epoch_info(self).map_err(|err| err.into())
+    }
+
+    fn get_balance(&self, pubkey: &Pubkey) -> Result<u64> {
+        RpcClient::get_balance(self, pubkey).map_err(|err| err.into())
+    }
+
+    fn get_balance_with_commitment(
+        &self,
+        pubkey: &Pubkey,
+        commitment_config: CommitmentConfig,
+    ) -> Result<u64> {
+        RpcClient::get_balance_with_commitment(self, pubkey, commitment_config)
+            .map(|res| res.value)
+            .map_err(|err| err.into())
+    }
+
+    fn get_fee_for_message(&self, message: &Message) -> Result<u64> {
+        RpcClient::get_fee_for_message(self, message).map_err(|err| err.into())
+    }
+
+    fn get_minimum_balance_for_rent_exemption(&self, data_len: usize) -> Result<u64> {
+        RpcClient::get_minimum_balance_for_rent_exemption(self, data_len).map_err(|err| err.into())
+    }
+
+    fn addr(&self) -> String {
+        self.url()
+    }
+
+    fn request_airdrop_with_blockhash(
+        &self,
+        pubkey: &Pubkey,
+        lamports: u64,
+        recent_blockhash: &Hash,
+    ) -> Result<Signature> {
+        RpcClient::request_airdrop_with_blockhash(self, pubkey, lamports, recent_blockhash)
+            .map_err(|err| err.into())
+    }
+}

bench-tps/src/bench_tps_client/thin_client.rs

@@ -0,0 +1,86 @@
+use {
+    crate::bench_tps_client::{BenchTpsClient, Result},
+    solana_client::thin_client::ThinClient,
+    solana_sdk::{
+        client::{AsyncClient, Client, SyncClient},
+        commitment_config::CommitmentConfig,
+        epoch_info::EpochInfo,
+        hash::Hash,
+        message::Message,
+        pubkey::Pubkey,
+        signature::Signature,
+        transaction::Transaction,
+    },
+};
+
+impl BenchTpsClient for ThinClient {
+    fn send_transaction(&self, transaction: Transaction) -> Result<Signature> {
+        AsyncClient::async_send_transaction(self, transaction).map_err(|err| err.into())
+    }
+    fn send_batch(&self, transactions: Vec<Transaction>) -> Result<()> {
+        AsyncClient::async_send_batch(self, transactions).map_err(|err| err.into())
+    }
+    fn get_latest_blockhash(&self) -> Result<Hash> {
+        SyncClient::get_latest_blockhash(self).map_err(|err| err.into())
+    }
+
+    fn get_latest_blockhash_with_commitment(
+        &self,
+        commitment_config: CommitmentConfig,
+    ) -> Result<(Hash, u64)> {
+        SyncClient::get_latest_blockhash_with_commitment(self, commitment_config)
+            .map_err(|err| err.into())
+    }
+
+    fn get_transaction_count(&self) -> Result<u64> {
+        SyncClient::get_transaction_count(self).map_err(|err| err.into())
+    }
+
+    fn get_transaction_count_with_commitment(
+        &self,
+        commitment_config: CommitmentConfig,
+    ) -> Result<u64> {
+        SyncClient::get_transaction_count_with_commitment(self, commitment_config)
+            .map_err(|err| err.into())
+    }
+
+    fn get_epoch_info(&self) -> Result<EpochInfo> {
+        SyncClient::get_epoch_info(self).map_err(|err| err.into())
+    }
+
+    fn get_balance(&self, pubkey: &Pubkey) -> Result<u64> {
+        SyncClient::get_balance(self, pubkey).map_err(|err| err.into())
+    }
+
+    fn get_balance_with_commitment(
+        &self,
+        pubkey: &Pubkey,
+        commitment_config: CommitmentConfig,
+    ) -> Result<u64> {
+        SyncClient::get_balance_with_commitment(self, pubkey, commitment_config)
+            .map_err(|err| err.into())
+    }
+
+    fn get_fee_for_message(&self, message: &Message) -> Result<u64> {
+        SyncClient::get_fee_for_message(self, message).map_err(|err| err.into())
+    }
+
+    fn get_minimum_balance_for_rent_exemption(&self, data_len: usize) -> Result<u64> {
+        SyncClient::get_minimum_balance_for_rent_exemption(self, data_len).map_err(|err| err.into())
+    }
+
+    fn addr(&self) -> String {
+        Client::tpu_addr(self)
+    }
+
+    fn request_airdrop_with_blockhash(
+        &self,
+        pubkey: &Pubkey,
+        lamports: u64,
+        recent_blockhash: &Hash,
+    ) -> Result<Signature> {
+        self.rpc_client()
+            .request_airdrop_with_blockhash(pubkey, lamports, recent_blockhash)
+            .map_err(|err| err.into())
+    }
+}

bench-tps/src/bench_tps_client/tpu_client.rs

@@ -0,0 +1,99 @@
+use {
+    crate::bench_tps_client::{BenchTpsClient, Result},
+    solana_client::tpu_client::TpuClient,
+    solana_sdk::{
+        commitment_config::CommitmentConfig, epoch_info::EpochInfo, hash::Hash, message::Message,
+        pubkey::Pubkey, signature::Signature, transaction::Transaction,
+    },
+};
+
+impl BenchTpsClient for TpuClient {
+    fn send_transaction(&self, transaction: Transaction) -> Result<Signature> {
+        let signature = transaction.signatures[0];
+        self.try_send_transaction(&transaction)?;
+        Ok(signature)
+    }
+    fn send_batch(&self, transactions: Vec<Transaction>) -> Result<()> {
+        for transaction in transactions {
+            BenchTpsClient::send_transaction(self, transaction)?;
+        }
+        Ok(())
+    }
+    fn get_latest_blockhash(&self) -> Result<Hash> {
+        self.rpc_client()
+            .get_latest_blockhash()
+            .map_err(|err| err.into())
+    }
+
+    fn get_latest_blockhash_with_commitment(
+        &self,
+        commitment_config: CommitmentConfig,
+    ) -> Result<(Hash, u64)> {
+        self.rpc_client()
+            .get_latest_blockhash_with_commitment(commitment_config)
+            .map_err(|err| err.into())
+    }
+
+    fn get_transaction_count(&self) -> Result<u64> {
+        self.rpc_client()
+            .get_transaction_count()
+            .map_err(|err| err.into())
+    }
+
+    fn get_transaction_count_with_commitment(
+        &self,
+        commitment_config: CommitmentConfig,
+    ) -> Result<u64> {
+        self.rpc_client()
+            .get_transaction_count_with_commitment(commitment_config)
+            .map_err(|err| err.into())
+    }
+
+    fn get_epoch_info(&self) -> Result<EpochInfo> {
+        self.rpc_client().get_epoch_info().map_err(|err| err.into())
+    }
+
+    fn get_balance(&self, pubkey: &Pubkey) -> Result<u64> {
+        self.rpc_client()
+            .get_balance(pubkey)
+            .map_err(|err| err.into())
+    }
+
+    fn get_balance_with_commitment(
+        &self,
+        pubkey: &Pubkey,
+        commitment_config: CommitmentConfig,
+    ) -> Result<u64> {
+        self.rpc_client()
+            .get_balance_with_commitment(pubkey, commitment_config)
+            .map(|res| res.value)
+            .map_err(|err| err.into())
+    }
+
+    fn get_fee_for_message(&self, message: &Message) -> Result<u64> {
+        self.rpc_client()
+            .get_fee_for_message(message)
+            .map_err(|err| err.into())
+    }
+
+    fn get_minimum_balance_for_rent_exemption(&self, data_len: usize) -> Result<u64> {
+        self.rpc_client()
+            .get_minimum_balance_for_rent_exemption(data_len)
+            .map_err(|err| err.into())
+    }
+
+    fn addr(&self) -> String {
+        self.rpc_client().url()
+    }
+
+    fn request_airdrop_with_blockhash(
+        &self,
+        pubkey: &Pubkey,
+        lamports: u64,
+        recent_blockhash: &Hash,
+    ) -> Result<Signature> {
+        self.rpc_client()
+            .request_airdrop_with_blockhash(pubkey, lamports, recent_blockhash)
+            .map_err(|err| err.into())
+    }
+}

bench-tps/src/cli.rs

@@ -1,6 +1,7 @@
 use {
     clap::{crate_description, crate_name, App, Arg, ArgMatches},
-    solana_faucet::faucet::FAUCET_PORT,
+    solana_clap_utils::input_validators::{is_url, is_url_or_moniker},
+    solana_cli_config::{ConfigInput, CONFIG_FILE},
     solana_sdk::{
         fee_calculator::FeeRateGovernor,
         pubkey::Pubkey,
@@ -11,10 +12,28 @@ use {
 
 const NUM_LAMPORTS_PER_ACCOUNT_DEFAULT: u64 = solana_sdk::native_token::LAMPORTS_PER_SOL;
 
+pub enum ExternalClientType {
+    // Submits transactions to an Rpc node using an RpcClient
+    RpcClient,
+    // Submits transactions directly to leaders using a ThinClient, broadcasting to multiple
+    // leaders when num_nodes > 1
+    ThinClient,
+    // Submits transactions directly to leaders using a TpuClient, broadcasting to upcoming leaders
+    // via TpuClient default configuration
+    TpuClient,
+}
+
+impl Default for ExternalClientType {
+    fn default() -> Self {
+        Self::ThinClient
+    }
+}
+
 /// Holds the configuration for a single run of the benchmark
 pub struct Config {
     pub entrypoint_addr: SocketAddr,
-    pub faucet_addr: SocketAddr,
+    pub json_rpc_url: String,
+    pub websocket_url: String,
     pub id: Keypair,
     pub threads: usize,
     pub num_nodes: usize,
@@ -31,13 +50,16 @@ pub struct Config {
     pub num_lamports_per_account: u64,
     pub target_slots_per_epoch: u64,
     pub target_node: Option<Pubkey>,
+    pub external_client_type: ExternalClientType,
+    pub use_quic: bool,
 }
 
 impl Default for Config {
     fn default() -> Config {
         Config {
             entrypoint_addr: SocketAddr::from(([127, 0, 0, 1], 8001)),
-            faucet_addr: SocketAddr::from(([127, 0, 0, 1], FAUCET_PORT)),
+            json_rpc_url: ConfigInput::default().json_rpc_url,
+            websocket_url: ConfigInput::default().websocket_url,
             id: Keypair::new(),
             threads: 4,
             num_nodes: 1,
@@ -54,6 +76,8 @@ impl Default for Config {
             num_lamports_per_account: NUM_LAMPORTS_PER_ACCOUNT_DEFAULT,
             target_slots_per_epoch: 0,
             target_node: None,
+            external_client_type: ExternalClientType::default(),
+            use_quic: false,
         }
     }
 }
@@ -62,6 +86,42 @@ impl Default for Config {
 pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> {
     App::new(crate_name!()).about(crate_description!())
         .version(version)
+        .arg({
+            let arg = Arg::with_name("config_file")
+                .short("C")
+                .long("config")
+                .value_name("FILEPATH")
+                .takes_value(true)
+                .global(true)
+                .help("Configuration file to use");
+            if let Some(ref config_file) = *CONFIG_FILE {
+                arg.default_value(config_file)
+            } else {
+                arg
+            }
+        })
+        .arg(
+            Arg::with_name("json_rpc_url")
+                .short("u")
+                .long("url")
+                .value_name("URL_OR_MONIKER")
+                .takes_value(true)
+                .global(true)
+                .validator(is_url_or_moniker)
+                .help(
+                    "URL for Solana's JSON RPC or moniker (or their first letter): \
+                       [mainnet-beta, testnet, devnet, localhost]",
+                ),
+        )
+        .arg(
+            Arg::with_name("websocket_url")
+                .long("ws")
+                .value_name("URL")
+                .takes_value(true)
+                .global(true)
+                .validator(is_url)
+                .help("WebSocket URL for the solana cluster"),
+        )
         .arg(
             Arg::with_name("entrypoint")
                 .short("n")
@@ -76,7 +136,8 @@ pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> {
                 .long("faucet")
                 .value_name("HOST:PORT")
                 .takes_value(true)
-                .help("Location of the faucet; defaults to entrypoint:FAUCET_PORT"),
+                .hidden(true)
+                .help("Deprecated. BenchTps no longer queries the faucet directly"),
         )
         .arg(
             Arg::with_name("identity")
@@ -191,6 +252,27 @@ pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> {
                     "Wait until epochs are this many slots long.",
                 ),
         )
+        .arg(
+            Arg::with_name("rpc_client")
+                .long("use-rpc-client")
+                .conflicts_with("tpu_client")
+                .takes_value(false)
+                .help("Submit transactions with a RpcClient")
+        )
+        .arg(
+            Arg::with_name("tpu_client")
+                .long("use-tpu-client")
+                .conflicts_with("rpc_client")
+                .takes_value(false)
+                .help("Submit transactions with a TpuClient")
+        )
+        .arg(
+            Arg::with_name("tpu_use_quic")
+                .long("tpu-use-quic")
+                .takes_value(false)
+                .help("Submit transactions via QUIC; only affects ThinClient (default) \
+                    or TpuClient sends"),
+        )
 }
 
 /// Parses a clap `ArgMatches` structure into a `Config`
@@ -201,6 +283,45 @@ pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> {
 pub fn extract_args(matches: &ArgMatches) -> Config {
     let mut args = Config::default();
 
+    let config = if let Some(config_file) = matches.value_of("config_file") {
+        solana_cli_config::Config::load(config_file).unwrap_or_default()
+    } else {
+        solana_cli_config::Config::default()
+    };
+    let (_, json_rpc_url) = ConfigInput::compute_json_rpc_url_setting(
+        matches.value_of("json_rpc_url").unwrap_or(""),
+        &config.json_rpc_url,
+    );
+    args.json_rpc_url = json_rpc_url;
+
+    let (_, websocket_url) = ConfigInput::compute_websocket_url_setting(
+        matches.value_of("websocket_url").unwrap_or(""),
+        &config.websocket_url,
+        matches.value_of("json_rpc_url").unwrap_or(""),
+        &config.json_rpc_url,
+    );
+    args.websocket_url = websocket_url;
+
+    let (_, id_path) = ConfigInput::compute_keypair_path_setting(
+        matches.value_of("identity").unwrap_or(""),
+        &config.keypair_path,
+    );
+    if let Ok(id) = read_keypair_file(id_path) {
+        args.id = id;
+    } else if matches.is_present("identity") {
+        panic!("could not parse identity path");
+    }
+
+    if matches.is_present("tpu_client") {
+        args.external_client_type = ExternalClientType::TpuClient;
+    } else if matches.is_present("rpc_client") {
+        args.external_client_type = ExternalClientType::RpcClient;
+    }
+
+    if matches.is_present("tpu_use_quic") {
+        args.use_quic = true;
+    }
+
     if let Some(addr) = matches.value_of("entrypoint") {
         args.entrypoint_addr = solana_net_utils::parse_host_port(addr).unwrap_or_else(|e| {
             eprintln!("failed to parse entrypoint address: {}", e);
@@ -208,18 +329,6 @@ pub fn extract_args(matches: &ArgMatches) -> Config {
         });
     }
 
-    if let Some(addr) = matches.value_of("faucet") {
-        args.faucet_addr = solana_net_utils::parse_host_port(addr).unwrap_or_else(|e| {
-            eprintln!("failed to parse faucet address: {}", e);
-            exit(1)
-        });
-    }
-
-    if matches.is_present("identity") {
-        args.id = read_keypair_file(matches.value_of("identity").unwrap())
-            .expect("can't read client identity");
-    }
-
     if let Some(t) = matches.value_of("threads") {
         args.threads = t.to_string().parse().expect("can't parse threads");
     }

bench-tps/src/keypairs.rs

@@ -0,0 +1,72 @@
+use {
+    crate::{
+        bench::{fund_keypairs, generate_and_fund_keypairs},
+        bench_tps_client::BenchTpsClient,
+    },
+    log::*,
+    solana_genesis::Base64Account,
+    solana_sdk::signature::{Keypair, Signer},
+    std::{collections::HashMap, fs::File, path::Path, process::exit, sync::Arc},
+};
+
+pub fn get_keypairs<T>(
+    client: Arc<T>,
+    id: &Keypair,
+    keypair_count: usize,
+    num_lamports_per_account: u64,
+    client_ids_and_stake_file: &str,
+    read_from_client_file: bool,
+) -> Vec<Keypair>
+where
+    T: 'static + BenchTpsClient + Send + Sync,
+{
+    if read_from_client_file {
+        let path = Path::new(client_ids_and_stake_file);
+        let file = File::open(path).unwrap();
+
+        info!("Reading {}", client_ids_and_stake_file);
+        let accounts: HashMap<String, Base64Account> = serde_yaml::from_reader(file).unwrap();
+        let mut keypairs = vec![];
+        let mut last_balance = 0;
+
+        accounts
+            .into_iter()
+            .for_each(|(keypair, primordial_account)| {
+                let bytes: Vec<u8> = serde_json::from_str(keypair.as_str()).unwrap();
+                keypairs.push(Keypair::from_bytes(&bytes).unwrap());
+                last_balance = primordial_account.balance;
+            });
+
+        if keypairs.len() < keypair_count {
+            eprintln!(
+                "Expected {} accounts in {}, only received {} (--tx_count mismatch?)",
+                keypair_count,
+                client_ids_and_stake_file,
+                keypairs.len(),
+            );
+            exit(1);
+        }
+        // Sort keypairs so that do_bench_tps() uses the same subset of accounts for each run.
+        // This prevents the amount of storage needed for bench-tps accounts from creeping up
+        // across multiple runs.
+        keypairs.sort_by_key(|x| x.pubkey().to_string());
+        fund_keypairs(
+            client,
+            id,
+            &keypairs,
+            keypairs.len().saturating_sub(keypair_count) as u64,
+            last_balance,
+        )
+        .unwrap_or_else(|e| {
+            eprintln!("Error could not fund keys: {:?}", e);
+            exit(1);
+        });
+        keypairs
+    } else {
+        generate_and_fund_keypairs(client, id, keypair_count, num_lamports_per_account)
+            .unwrap_or_else(|e| {
+                eprintln!("Error could not fund keys: {:?}", e);
+                exit(1);
+            })
+    }
+}

bench-tps/src/lib.rs

@@ -1,3 +1,6 @@
 #![allow(clippy::integer_arithmetic)]
 pub mod bench;
+pub mod bench_tps_client;
 pub mod cli;
+pub mod keypairs;
+mod perf_utils;

bench-tps/src/main.rs

@@ -2,15 +2,19 @@
 use {
     log::*,
     solana_bench_tps::{
-        bench::{do_bench_tps, generate_and_fund_keypairs, generate_keypairs},
-        cli,
+        bench::{do_bench_tps, generate_keypairs},
+        cli::{self, ExternalClientType},
+        keypairs::get_keypairs,
+    },
+    solana_client::{
+        connection_cache,
+        rpc_client::RpcClient,
+        tpu_client::{TpuClient, TpuClientConfig},
     },
     solana_genesis::Base64Account,
     solana_gossip::gossip_service::{discover_cluster, get_client, get_multi_client},
     solana_sdk::{
-        fee_calculator::FeeRateGovernor,
-        signature::{Keypair, Signer},
-        system_program,
+        commitment_config::CommitmentConfig, fee_calculator::FeeRateGovernor, system_program,
     },
     solana_streamer::socket::SocketAddrSpace,
     std::{collections::HashMap, fs::File, io::prelude::*, path::Path, process::exit, sync::Arc},
@@ -28,7 +32,8 @@ fn main() {
 
     let cli::Config {
         entrypoint_addr,
-        faucet_addr,
+        json_rpc_url,
+        websocket_url,
         id,
         num_nodes,
         tx_count,
@@ -40,6 +45,8 @@ fn main() {
         multi_client,
         num_lamports_per_account,
         target_node,
+        external_client_type,
+        use_quic,
         ..
     } = &cli_config;
 
@@ -75,12 +82,32 @@ fn main() {
     }
 
     info!("Connecting to the cluster");
+
+    match external_client_type {
+        ExternalClientType::RpcClient => {
+            let client = Arc::new(RpcClient::new_with_commitment(
+                json_rpc_url.to_string(),
+                CommitmentConfig::confirmed(),
+            ));
+            let keypairs = get_keypairs(
+                client.clone(),
+                id,
+                keypair_count,
+                *num_lamports_per_account,
+                client_ids_and_stake_file,
+                *read_from_client_file,
+            );
+            do_bench_tps(client, cli_config, keypairs);
+        }
+        ExternalClientType::ThinClient => {
             let nodes = discover_cluster(entrypoint_addr, *num_nodes, SocketAddrSpace::Unspecified)
                 .unwrap_or_else(|err| {
                     eprintln!("Failed to discover {} nodes: {:?}", num_nodes, err);
                     exit(1);
                 });
-
+            if *use_quic {
+                connection_cache::set_use_quic(true);
+            }
             let client = if *multi_client {
                 let (client, num_clients) = get_multi_client(&nodes, &SocketAddrSpace::Unspecified);
                 if nodes.len() < num_clients {
@@ -96,7 +123,8 @@ fn main() {
                 let mut target_client = None;
                 for node in nodes {
                     if node.id == *target_node {
-                target_client = Some(Arc::new(get_client(&[node], &SocketAddrSpace::Unspecified)));
+                        target_client =
+                            Some(Arc::new(get_client(&[node], &SocketAddrSpace::Unspecified)));
                         break;
                     }
                 }
@@ -107,51 +135,40 @@ fn main() {
             } else {
                 Arc::new(get_client(&nodes, &SocketAddrSpace::Unspecified))
             };
-
-    let keypairs = if *read_from_client_file {
-        let path = Path::new(&client_ids_and_stake_file);
-        let file = File::open(path).unwrap();
-
-        info!("Reading {}", client_ids_and_stake_file);
-        let accounts: HashMap<String, Base64Account> = serde_yaml::from_reader(file).unwrap();
-        let mut keypairs = vec![];
-        let mut last_balance = 0;
-
-        accounts
-            .into_iter()
-            .for_each(|(keypair, primordial_account)| {
-                let bytes: Vec<u8> = serde_json::from_str(keypair.as_str()).unwrap();
-                keypairs.push(Keypair::from_bytes(&bytes).unwrap());
-                last_balance = primordial_account.balance;
-            });
-
-        if keypairs.len() < keypair_count {
-            eprintln!(
-                "Expected {} accounts in {}, only received {} (--tx_count mismatch?)",
-                keypair_count,
-                client_ids_and_stake_file,
-                keypairs.len(),
-            );
-            exit(1);
-        }
-        // Sort keypairs so that do_bench_tps() uses the same subset of accounts for each run.
-        // This prevents the amount of storage needed for bench-tps accounts from creeping up
-        // across multiple runs.
-        keypairs.sort_by_key(|x| x.pubkey().to_string());
-        keypairs
-    } else {
-        generate_and_fund_keypairs(
+            let keypairs = get_keypairs(
                 client.clone(),
-            Some(*faucet_addr),
                 id,
                 keypair_count,
                 *num_lamports_per_account,
-        )
-        .unwrap_or_else(|e| {
-            eprintln!("Error could not fund keys: {:?}", e);
-            exit(1);
-        })
-    };
-
+                client_ids_and_stake_file,
+                *read_from_client_file,
+            );
             do_bench_tps(client, cli_config, keypairs);
+        }
+        ExternalClientType::TpuClient => {
+            let rpc_client = Arc::new(RpcClient::new_with_commitment(
+                json_rpc_url.to_string(),
+                CommitmentConfig::confirmed(),
+            ));
+            if *use_quic {
+                connection_cache::set_use_quic(true);
+            }
+            let client = Arc::new(
+                TpuClient::new(rpc_client, websocket_url, TpuClientConfig::default())
+                    .unwrap_or_else(|err| {
+                        eprintln!("Could not create TpuClient {:?}", err);
+                        exit(1);
+                    }),
+            );
+            let keypairs = get_keypairs(
+                client.clone(),
+                id,
+                keypair_count,
+                *num_lamports_per_account,
+                client_ids_and_stake_file,
+                *read_from_client_file,
+            );
+            do_bench_tps(client, cli_config, keypairs);
+        }
+    }
 }

bench-tps/src/perf_utils.rs

@@ -1,6 +1,7 @@
 use {
+    crate::bench_tps_client::BenchTpsClient,
     log::*,
-    solana_sdk::{client::Client, commitment_config::CommitmentConfig, timing::duration_as_s},
+    solana_sdk::{commitment_config::CommitmentConfig, timing::duration_as_s},
     std::{
         sync::{
             atomic::{AtomicBool, Ordering},
@@ -27,7 +28,7 @@ pub fn sample_txs<T>(
     sample_period: u64,
     client: &Arc<T>,
 ) where
-    T: Client,
+    T: BenchTpsClient,
 {
     let mut max_tps = 0.0;
     let mut total_elapsed;
@@ -81,10 +82,7 @@ pub fn sample_txs<T>(
                 elapsed: total_elapsed,
                 txs: total_txs,
             };
-            sample_stats
-                .write()
-                .unwrap()
-                .push((client.tpu_addr(), stats));
+            sample_stats.write().unwrap().push((client.addr(), stats));
             return;
         }
         sleep(Duration::from_secs(sample_period));

bench-tps/tests/bench_tps.rs

@@ -6,16 +6,24 @@ use {
         bench::{do_bench_tps, generate_and_fund_keypairs},
         cli::Config,
     },
-    solana_client::thin_client::create_client,
+    solana_client::{
+        rpc_client::RpcClient,
+        thin_client::create_client,
+        tpu_client::{TpuClient, TpuClientConfig},
+    },
     solana_core::validator::ValidatorConfig,
-    solana_faucet::faucet::run_local_faucet_with_port,
-    solana_gossip::cluster_info::VALIDATOR_PORT_RANGE,
+    solana_faucet::faucet::{run_local_faucet, run_local_faucet_with_port},
     solana_local_cluster::{
         local_cluster::{ClusterConfig, LocalCluster},
         validator_configs::make_identical_validator_configs,
     },
-    solana_sdk::signature::{Keypair, Signer},
+    solana_rpc::rpc::JsonRpcConfig,
+    solana_sdk::{
+        commitment_config::CommitmentConfig,
+        signature::{Keypair, Signer},
+    },
     solana_streamer::socket::SocketAddrSpace,
+    solana_test_validator::TestValidator,
     std::{sync::Arc, time::Duration},
 };
 
@@ -23,13 +31,34 @@ fn test_bench_tps_local_cluster(config: Config) {
     let native_instruction_processors = vec![];
 
     solana_logger::setup();
+
+    let faucet_keypair = Keypair::new();
+    let faucet_pubkey = faucet_keypair.pubkey();
+    let (addr_sender, addr_receiver) = unbounded();
+    run_local_faucet_with_port(faucet_keypair, addr_sender, None, 0);
+    let faucet_addr = addr_receiver
+        .recv_timeout(Duration::from_secs(2))
+        .expect("run_local_faucet")
+        .expect("faucet_addr");
+
     const NUM_NODES: usize = 1;
+    let mut validator_config = ValidatorConfig::default_for_test();
+    validator_config.rpc_config = JsonRpcConfig {
+        faucet_addr: Some(faucet_addr),
+        ..JsonRpcConfig::default_for_test()
+    };
     let cluster = LocalCluster::new(
         &mut ClusterConfig {
             node_stakes: vec![999_990; NUM_NODES],
             cluster_lamports: 200_000_000,
             validator_configs: make_identical_validator_configs(
-                &ValidatorConfig::default_for_test(),
+                &ValidatorConfig {
+                    rpc_config: JsonRpcConfig {
+                        faucet_addr: Some(faucet_addr),
+                        ..JsonRpcConfig::default_for_test()
+                    },
+                    ..ValidatorConfig::default_for_test()
+                },
                 NUM_NODES,
             ),
             native_instruction_processors,
@@ -38,31 +67,55 @@ fn test_bench_tps_local_cluster(config: Config) {
         SocketAddrSpace::Unspecified,
     );
 
-    let faucet_keypair = Keypair::new();
-    cluster.transfer(
-        &cluster.funding_keypair,
-        &faucet_keypair.pubkey(),
-        100_000_000,
-    );
+    cluster.transfer(&cluster.funding_keypair, &faucet_pubkey, 100_000_000);
 
     let client = Arc::new(create_client(
-        (cluster.entry_point_info.rpc, cluster.entry_point_info.tpu),
-        VALIDATOR_PORT_RANGE,
+        cluster.entry_point_info.rpc,
+        cluster.entry_point_info.tpu,
     ));
 
-    let (addr_sender, addr_receiver) = unbounded();
-    run_local_faucet_with_port(faucet_keypair, addr_sender, None, 0);
-    let faucet_addr = addr_receiver
-        .recv_timeout(Duration::from_secs(2))
-        .expect("run_local_faucet")
-        .expect("faucet_addr");
-
     let lamports_per_account = 100;
 
     let keypair_count = config.tx_count * config.keypair_multiplier;
     let keypairs = generate_and_fund_keypairs(
         client.clone(),
-        Some(faucet_addr),
+        &config.id,
+        keypair_count,
+        lamports_per_account,
+    )
+    .unwrap();
+
+    let _total = do_bench_tps(client, config, keypairs);
+
+    #[cfg(not(debug_assertions))]
+    assert!(_total > 100);
+}
+
+fn test_bench_tps_test_validator(config: Config) {
+    solana_logger::setup();
+
+    let mint_keypair = Keypair::new();
+    let mint_pubkey = mint_keypair.pubkey();
+
+    let faucet_addr = run_local_faucet(mint_keypair, None);
+
+    let test_validator =
+        TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr), SocketAddrSpace::Unspecified);
+
+    let rpc_client = Arc::new(RpcClient::new_with_commitment(
+        test_validator.rpc_url(),
+        CommitmentConfig::processed(),
+    ));
+    let websocket_url = test_validator.rpc_pubsub_url();
+
+    let client =
+        Arc::new(TpuClient::new(rpc_client, &websocket_url, TpuClientConfig::default()).unwrap());
+
+    let lamports_per_account = 100;
+
+    let keypair_count = config.tx_count * config.keypair_multiplier;
+    let keypairs = generate_and_fund_keypairs(
+        client.clone(),
         &config.id,
         keypair_count,
         lamports_per_account,
@@ -84,3 +137,13 @@ fn test_bench_tps_local_cluster_solana() {
         ..Config::default()
     });
 }
+
+#[test]
+#[serial]
+fn test_bench_tps_tpu_client() {
+    test_bench_tps_test_validator(Config {
+        tx_count: 100,
+        duration: Duration::from_secs(10),
+        ..Config::default()
+    });
+}

bloom/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bloom"
-version = "1.10.3"
+version = "1.11.0"
 description = "Solana bloom filter"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -17,9 +17,9 @@ rand = "0.7.0"
 rayon = "1.5.1"
 serde = { version = "1.0.136", features = ["rc"] }
 serde_derive = "1.0.103"
-solana-frozen-abi = { path = "../frozen-abi", version = "=1.10.3" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.10.3" }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.11.0" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
 
 [lib]
 crate-type = ["lib"]

bucket_map/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bucket-map"
-version = "1.10.3"
+version = "1.11.0"
 description = "solana-bucket-map"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-bucket-map"
@@ -15,14 +15,14 @@ log = { version = "0.4.11" }
 memmap2 = "0.5.3"
 modular-bitfield = "0.11.2"
 rand = "0.7.0"
-solana-measure = { path = "../measure", version = "=1.10.3" }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
+solana-measure = { path = "../measure", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
 tempfile = "3.3.0"
 
 [dev-dependencies]
 fs_extra = "1.2.0"
 rayon = "1.5.0"
-solana-logger = { path = "../logger", version = "=1.10.3" }
+solana-logger = { path = "../logger", version = "=1.11.0" }
 
 [lib]
 crate-type = ["lib"]

bucket_map/src/index_entry.rs

@@ -1,4 +1,5 @@
 #![allow(dead_code)]
+
 use {
     crate::{
         bucket::Bucket,
@@ -57,8 +58,18 @@ impl IndexEntry {
             .expect("New storage offset must fit into 7 bytes!")
     }
 
+    /// return closest bucket index fit for the slot slice.
+    /// Since bucket size is 2^index, the return value is
+    ///     min index, such that 2^index >= num_slots
+    ///     index = ceiling(log2(num_slots))
+    /// special case, when slot slice empty, return 0th index.
     pub fn data_bucket_from_num_slots(num_slots: Slot) -> u64 {
-        (num_slots as f64).log2().ceil() as u64 // use int log here?
+        // Compute the ceiling of log2 for integer
+        if num_slots == 0 {
+            0
+        } else {
+            (Slot::BITS - (num_slots - 1).leading_zeros()) as u64
+        }
     }
 
     pub fn data_bucket_ix(&self) -> u64 {
@@ -153,4 +164,23 @@ mod tests {
         let mut index = IndexEntry::new(Pubkey::new_unique());
         index.set_storage_offset(too_big);
     }
+
+    #[test]
+    fn test_data_bucket_from_num_slots() {
+        for n in 0..512 {
+            assert_eq!(
+                IndexEntry::data_bucket_from_num_slots(n),
+                (n as f64).log2().ceil() as u64
+            );
+        }
+        assert_eq!(IndexEntry::data_bucket_from_num_slots(u32::MAX as u64), 32);
+        assert_eq!(
+            IndexEntry::data_bucket_from_num_slots(u32::MAX as u64 + 1),
+            32
+        );
+        assert_eq!(
+            IndexEntry::data_bucket_from_num_slots(u32::MAX as u64 + 2),
+            33
+        );
+    }
 }

ci/buildkite-pipeline-in-disk.sh

@@ -137,7 +137,7 @@ all_test_steps() {
              ^ci/test-coverage.sh \
              ^scripts/coverage.sh \
       ; then
-    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 40
+    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 50
     wait_step
   else
     annotate --style info --context test-coverage \
@@ -152,14 +152,14 @@ all_test_steps() {
              ^ci/test-coverage.sh \
              ^scripts/coverage-in-disk.sh \
       ; then
-    command_step coverage-in-disk ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 40
+    command_step coverage-in-disk ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 50
     wait_step
   else
     annotate --style info --context test-coverage \
       "Coverage skipped as no .rs files were modified"
   fi
   # Full test suite
-  command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 60
+  command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 70
   wait_step
 
   # BPF test suite
@@ -303,7 +303,7 @@ EOF
 
   command_step "local-cluster-slow" \
     ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster-slow.sh" \
-    30
+    40
 }
 
 pull_or_push_steps() {

ci/buildkite-pipeline.sh

@@ -139,7 +139,7 @@ all_test_steps() {
              ^ci/test-coverage.sh \
              ^scripts/coverage.sh \
       ; then
-    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 40
+    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 50
     wait_step
   else
     annotate --style info --context test-coverage \
@@ -147,7 +147,7 @@ all_test_steps() {
   fi
 
   # Full test suite
-  command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 60
+  command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 70
   wait_step
 
   # BPF test suite
@@ -295,7 +295,7 @@ EOF
 
   command_step "local-cluster-slow" \
     ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster-slow.sh" \
-    30
+    40
 }
 
 pull_or_push_steps() {

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.59.0
+FROM solanalabs/rust:1.60.0
 ARG date
 
 RUN set -x \

ci/docker-rust-nightly/build.sh

@@ -3,8 +3,14 @@ set -ex
 
 cd "$(dirname "$0")"
 
+platform=()
+if [[ $(uname -m) = arm64 ]]; then
+  # Ref: https://blog.jaimyn.dev/how-to-build-multi-architecture-docker-images-on-an-m1-mac/#tldr
+  platform+=(--platform linux/amd64)
+fi
+
 nightlyDate=${1:-$(date +%Y-%m-%d)}
-docker build -t solanalabs/rust-nightly:"$nightlyDate" --build-arg date="$nightlyDate" .
+docker build "${platform[@]}" -t solanalabs/rust-nightly:"$nightlyDate" --build-arg date="$nightlyDate" .
 
 maybeEcho=
 if [[ -z $CI ]]; then

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.59.0
+FROM rust:1.60.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0

ci/docker-rust/build.sh

@@ -3,7 +3,14 @@ set -ex
 
 cd "$(dirname "$0")"
 
-docker build -t solanalabs/rust .
+
+platform=()
+if [[ $(uname -m) = arm64 ]]; then
+  # Ref: https://blog.jaimyn.dev/how-to-build-multi-architecture-docker-images-on-an-m1-mac/#tldr
+  platform+=(--platform linux/amd64)
+fi
+
+docker build "${platform[@]}" -t solanalabs/rust .
 
 read -r rustc version _ < <(docker run solanalabs/rust rustc --version)
 [[ $rustc = rustc ]]

ci/rust-version.sh

@@ -18,13 +18,13 @@
 if [[ -n $RUST_STABLE_VERSION ]]; then
   stable_version="$RUST_STABLE_VERSION"
 else
-  stable_version=1.59.0
+  stable_version=1.60.0
 fi
 
 if [[ -n $RUST_NIGHTLY_VERSION ]]; then
   nightly_version="$RUST_NIGHTLY_VERSION"
 else
-  nightly_version=2022-02-24
+  nightly_version=2022-04-01
 fi
 
 

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.10.3"
+version = "1.11.0"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,12 +13,12 @@ edition = "2021"
 chrono = "0.4"
 clap = "2.33.0"
 rpassword = "6.0"
-solana-perf = { path = "../perf", version = "=1.10.3" }
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.10.3", default-features = false }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
+solana-perf = { path = "../perf", version = "=1.11.0" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.11.0", default-features = false }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
 thiserror = "1.0.30"
 tiny-bip39 = "0.8.2"
-uriparse = "0.6.3"
+uriparse = "0.6.4"
 url = "2.2.2"
 
 [dev-dependencies]

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.10.3"
+version = "1.11.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,6 +15,8 @@ lazy_static = "1.4.0"
 serde = "1.0.136"
 serde_derive = "1.0.103"
 serde_yaml = "0.8.23"
+solana-clap-utils = { path = "../clap-utils", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
 url = "2.2.2"
 
 [dev-dependencies]

cli-config/src/config_input.rs

@@ -0,0 +1,126 @@
+use {
+    crate::Config, solana_clap_utils::input_validators::normalize_to_url_if_moniker,
+    solana_sdk::commitment_config::CommitmentConfig, std::str::FromStr,
+};
+
+pub enum SettingType {
+    Explicit,
+    Computed,
+    SystemDefault,
+}
+
+pub struct ConfigInput {
+    pub json_rpc_url: String,
+    pub websocket_url: String,
+    pub keypair_path: String,
+    pub commitment: CommitmentConfig,
+}
+
+impl ConfigInput {
+    fn default_keypair_path() -> String {
+        Config::default().keypair_path
+    }
+
+    fn default_json_rpc_url() -> String {
+        Config::default().json_rpc_url
+    }
+
+    fn default_websocket_url() -> String {
+        Config::default().websocket_url
+    }
+
+    fn default_commitment() -> CommitmentConfig {
+        CommitmentConfig::confirmed()
+    }
+
+    fn first_nonempty_setting(
+        settings: std::vec::Vec<(SettingType, String)>,
+    ) -> (SettingType, String) {
+        settings
+            .into_iter()
+            .find(|(_, value)| !value.is_empty())
+            .expect("no nonempty setting")
+    }
+
+    fn first_setting_is_some<T>(
+        settings: std::vec::Vec<(SettingType, Option<T>)>,
+    ) -> (SettingType, T) {
+        let (setting_type, setting_option) = settings
+            .into_iter()
+            .find(|(_, value)| value.is_some())
+            .expect("all settings none");
+        (setting_type, setting_option.unwrap())
+    }
+
+    pub fn compute_websocket_url_setting(
+        websocket_cmd_url: &str,
+        websocket_cfg_url: &str,
+        json_rpc_cmd_url: &str,
+        json_rpc_cfg_url: &str,
+    ) -> (SettingType, String) {
+        Self::first_nonempty_setting(vec![
+            (SettingType::Explicit, websocket_cmd_url.to_string()),
+            (SettingType::Explicit, websocket_cfg_url.to_string()),
+            (
+                SettingType::Computed,
+                Config::compute_websocket_url(&normalize_to_url_if_moniker(json_rpc_cmd_url)),
+            ),
+            (
+                SettingType::Computed,
+                Config::compute_websocket_url(&normalize_to_url_if_moniker(json_rpc_cfg_url)),
+            ),
+            (SettingType::SystemDefault, Self::default_websocket_url()),
+        ])
+    }
+
+    pub fn compute_json_rpc_url_setting(
+        json_rpc_cmd_url: &str,
+        json_rpc_cfg_url: &str,
+    ) -> (SettingType, String) {
+        let (setting_type, url_or_moniker) = Self::first_nonempty_setting(vec![
+            (SettingType::Explicit, json_rpc_cmd_url.to_string()),
+            (SettingType::Explicit, json_rpc_cfg_url.to_string()),
+            (SettingType::SystemDefault, Self::default_json_rpc_url()),
+        ]);
+        (setting_type, normalize_to_url_if_moniker(&url_or_moniker))
+    }
+
+    pub fn compute_keypair_path_setting(
+        keypair_cmd_path: &str,
+        keypair_cfg_path: &str,
+    ) -> (SettingType, String) {
+        Self::first_nonempty_setting(vec![
+            (SettingType::Explicit, keypair_cmd_path.to_string()),
+            (SettingType::Explicit, keypair_cfg_path.to_string()),
+            (SettingType::SystemDefault, Self::default_keypair_path()),
+        ])
+    }
+
+    pub fn compute_commitment_config(
+        commitment_cmd: &str,
+        commitment_cfg: &str,
+    ) -> (SettingType, CommitmentConfig) {
+        Self::first_setting_is_some(vec![
+            (
+                SettingType::Explicit,
+                CommitmentConfig::from_str(commitment_cmd).ok(),
+            ),
+            (
+                SettingType::Explicit,
+                CommitmentConfig::from_str(commitment_cfg).ok(),
+            ),
+            (SettingType::SystemDefault, Some(Self::default_commitment())),
+        ])
+    }
+}
+
+impl Default for ConfigInput {
+    fn default() -> ConfigInput {
+        ConfigInput {
+            json_rpc_url: Self::default_json_rpc_url(),
+            websocket_url: Self::default_websocket_url(),
+            keypair_path: Self::default_keypair_path(),
+            commitment: CommitmentConfig::confirmed(),
+        }
+    }
+}

cli-config/src/lib.rs

@@ -55,12 +55,16 @@
 extern crate lazy_static;
 
 mod config;
-pub use config::{Config, CONFIG_FILE};
+mod config_input;
 use std::{
     fs::{create_dir_all, File},
     io::{self, Write},
     path::Path,
 };
+pub use {
+    config::{Config, CONFIG_FILE},
+    config_input::{ConfigInput, SettingType},
+};
 
 /// Load a value from a file in YAML format.
 ///

cli-output/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.10.3"
+version = "1.11.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -17,14 +17,17 @@ clap = "2.33.0"
 console = "0.15.0"
 humantime = "2.0.1"
 indicatif = "0.16.2"
+pretty-hex = "0.2.1"
+semver = "1.0.6"
 serde = "1.0.136"
 serde_json = "1.0.79"
-solana-account-decoder = { path = "../account-decoder", version = "=1.10.3" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.10.3" }
-solana-client = { path = "../client", version = "=1.10.3" }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.3" }
-solana-vote-program = { path = "../programs/vote", version = "=1.10.3" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.11.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.11.0" }
+solana-cli-config = { path = "../cli-config", version = "=1.11.0" }
+solana-client = { path = "../client", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.11.0" }
+solana-vote-program = { path = "../programs/vote", version = "=1.11.0" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 
 [dev-dependencies]

cli-output/src/cli_output.rs

@@ -356,6 +356,7 @@ pub enum CliValidatorsSortOrder {
     SkipRate,
     Stake,
     VoteAccount,
+    Version,
 }
 
 #[derive(Serialize, Deserialize)]
@@ -494,6 +495,22 @@ impl fmt::Display for CliValidators {
             CliValidatorsSortOrder::Stake => {
                 sorted_validators.sort_by_key(|a| a.activated_stake);
             }
+            CliValidatorsSortOrder::Version => {
+                sorted_validators.sort_by(|a, b| {
+                    use std::cmp::Ordering;
+                    let a_version = semver::Version::parse(a.version.as_str()).ok();
+                    let b_version = semver::Version::parse(b.version.as_str()).ok();
+                    match (a_version, b_version) {
+                        (None, None) => a.version.cmp(&b.version),
+                        (None, Some(_)) => Ordering::Less,
+                        (Some(_), None) => Ordering::Greater,
+                        (Some(va), Some(vb)) => match va.cmp(&vb) {
+                            Ordering::Equal => a.activated_stake.cmp(&b.activated_stake),
+                            ordering => ordering,
+                        },
+                    }
+                });
+            }
         }
 
         if self.validators_reverse_sort {
@@ -2776,10 +2793,10 @@ mod tests {
 
         let expected_msg = "AwECBwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDgTl3Dqh9\
             F19Wo1Rmw0x+zMuNipG07jeiXfYPW4/Js5QEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE\
-            BAQEBAYGBgYGBgYGBgYGBgYGBgYGBgYGBgYGBgYGBgYGBgYGBQUFBQUFBQUFBQUFBQUFBQUF\
-            BQUFBQUFBQUFBQUFBQUGp9UXGSxWjuCKhF9z0peIzwNcMUWyGrNE2AYuqUAAAAAAAAAAAAAA\
-            AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcH\
-            BwcCBgMDBQIEBAAAAAYCAQQMAgAAACoAAAAAAAAA"
+            BAQEBAUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBgYGBgYGBgYGBgYGBgYGBgYG\
+            BgYGBgYGBgYGBgYGBgYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAan1RcZLFaO\
+            4IqEX3PSl4jPA1wxRbIas0TYBi6pQAAABwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcH\
+            BwcCBQMEBgIEBAAAAAUCAQMMAgAAACoAAAAAAAAA"
             .to_string();
         let config = ReturnSignersConfig {
             dump_transaction_message: true,

cli-output/src/display.rs

@@ -3,6 +3,7 @@ use {
     chrono::{DateTime, Local, NaiveDateTime, SecondsFormat, TimeZone, Utc},
     console::style,
     indicatif::{ProgressBar, ProgressStyle},
+    solana_cli_config::SettingType,
     solana_sdk::{
         clock::UnixTimestamp,
         hash::Hash,
@@ -14,6 +15,7 @@ use {
         signature::Signature,
         stake,
         transaction::{TransactionError, TransactionVersion, VersionedTransaction},
+        transaction_context::TransactionReturnData,
     },
     solana_transaction_status::{Rewards, UiTransactionStatusMeta},
     spl_memo::{id as spl_memo_id, v1::id as spl_memo_v1_id},
@@ -103,6 +105,21 @@ pub fn writeln_name_value(f: &mut dyn fmt::Write, name: &str, value: &str) -> fm
     writeln!(f, "{} {}", style(name).bold(), styled_value)
 }
 
+pub fn println_name_value_or(name: &str, value: &str, setting_type: SettingType) {
+    let description = match setting_type {
+        SettingType::Explicit => "",
+        SettingType::Computed => "(computed)",
+        SettingType::SystemDefault => "(default)",
+    };
+
+    println!(
+        "{} {} {}",
+        style(name).bold(),
+        style(value),
+        style(description).italic(),
+    );
+}
+
 pub fn format_labeled_address(pubkey: &str, address_labels: &HashMap<String, String>) -> String {
     let label = address_labels.get(pubkey);
     match label {
@@ -246,6 +263,7 @@ fn write_transaction<W: io::Write>(
         write_fees(w, transaction_status.fee, prefix)?;
         write_balances(w, transaction_status, prefix)?;
         write_log_messages(w, transaction_status.log_messages.as_ref(), prefix)?;
+        write_return_data(w, transaction_status.return_data.as_ref(), prefix)?;
         write_rewards(w, transaction_status.rewards.as_ref(), prefix)?;
     } else {
         writeln!(w, "{}Status: Unavailable", prefix)?;
@@ -576,6 +594,25 @@ fn write_balances<W: io::Write>(
     Ok(())
 }
 
+fn write_return_data<W: io::Write>(
+    w: &mut W,
+    return_data: Option<&TransactionReturnData>,
+    prefix: &str,
+) -> io::Result<()> {
+    if let Some(return_data) = return_data {
+        if !return_data.data.is_empty() {
+            use pretty_hex::*;
+            writeln!(
+                w,
+                "{}Return Data from Program {}:",
+                prefix, return_data.program_id
+            )?;
+            writeln!(w, "{}  {:?}", prefix, return_data.data.hex_dump())?;
+        }
+    }
+    Ok(())
+}
+
 fn write_log_messages<W: io::Write>(
     w: &mut W,
     log_messages: Option<&Vec<String>>,
@@ -750,6 +787,10 @@ mod test {
                 commission: None,
             }]),
             loaded_addresses: LoadedAddresses::default(),
+            return_data: Some(TransactionReturnData {
+                program_id: Pubkey::new_from_array([2u8; 32]),
+                data: vec![1, 2, 3],
+            }),
         };
 
         let output = {
@@ -786,6 +827,9 @@ Status: Ok
   Account 1 balance: ◎0.00001 -> ◎0.0000099
 Log Messages:
   Test message
+Return Data from Program 8qbHbw2BbbTHBW1sbeqakYXVKRQM8Ne7pLK7m6CVfeR:
+  Length: 3 (0x3) bytes
+0000:   01 02 03                                             ...
 Rewards:
   Address                                            Type        Amount            New Balance         \0
   4vJ9JU1bJJE96FWSJKvHsmmFADCg4gpZQff4P3bkLKi        rent        -◎0.000000100     ◎0.000009900       \0
@@ -820,6 +864,10 @@ Rewards:
                 commission: None,
             }]),
             loaded_addresses,
+            return_data: Some(TransactionReturnData {
+                program_id: Pubkey::new_from_array([2u8; 32]),
+                data: vec![1, 2, 3],
+            }),
         };
 
         let output = {
@@ -865,6 +913,9 @@ Status: Ok
   Account 3 balance: ◎0.00002
 Log Messages:
   Test message
+Return Data from Program 8qbHbw2BbbTHBW1sbeqakYXVKRQM8Ne7pLK7m6CVfeR:
+  Length: 3 (0x3) bytes
+0000:   01 02 03                                             ...
 Rewards:
   Address                                            Type        Amount            New Balance         \0
   CktRuQ2mttgRGkXJtyksdKHjUdc2C4TgDzyB98oEzy8        rent        -◎0.000000100     ◎0.000014900       \0

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.10.3"
+version = "1.11.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -27,29 +27,29 @@ semver = "1.0.6"
 serde = "1.0.136"
 serde_derive = "1.0.103"
 serde_json = "1.0.79"
-solana-account-decoder = { path = "../account-decoder", version = "=1.10.3" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.10.3" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.10.3" }
-solana-cli-config = { path = "../cli-config", version = "=1.10.3" }
-solana-cli-output = { path = "../cli-output", version = "=1.10.3" }
-solana-client = { path = "../client", version = "=1.10.3" }
-solana-config-program = { path = "../programs/config", version = "=1.10.3" }
-solana-faucet = { path = "../faucet", version = "=1.10.3" }
-solana-logger = { path = "../logger", version = "=1.10.3" }
-solana-program-runtime = { path = "../program-runtime", version = "=1.10.3" }
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.10.3" }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.3" }
-solana-version = { path = "../version", version = "=1.10.3" }
-solana-vote-program = { path = "../programs/vote", version = "=1.10.3" }
-solana_rbpf = "=0.2.24"
+solana-account-decoder = { path = "../account-decoder", version = "=1.11.0" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.11.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.11.0" }
+solana-cli-config = { path = "../cli-config", version = "=1.11.0" }
+solana-cli-output = { path = "../cli-output", version = "=1.11.0" }
+solana-client = { path = "../client", version = "=1.11.0" }
+solana-config-program = { path = "../programs/config", version = "=1.11.0" }
+solana-faucet = { path = "../faucet", version = "=1.11.0" }
+solana-logger = { path = "../logger", version = "=1.11.0" }
+solana-program-runtime = { path = "../program-runtime", version = "=1.11.0" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.11.0" }
+solana-version = { path = "../version", version = "=1.11.0" }
+solana-vote-program = { path = "../programs/vote", version = "=1.11.0" }
+solana_rbpf = "=0.2.25"
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 thiserror = "1.0.30"
 tiny-bip39 = "0.8.2"
 
 [dev-dependencies]
-solana-streamer = { path = "../streamer", version = "=1.10.3" }
-solana-test-validator = { path = "../test-validator", version = "=1.10.3" }
+solana-streamer = { path = "../streamer", version = "=1.11.0" }
+solana-test-validator = { path = "../test-validator", version = "=1.11.0" }
 tempfile = "3.3.0"
 
 [[bin]]

cli/src/cli.rs

@@ -7,7 +7,8 @@ use {
     log::*,
     num_traits::FromPrimitive,
     serde_json::{self, Value},
-    solana_clap_utils::{self, input_parsers::*, input_validators::*, keypair::*},
+    solana_clap_utils::{self, input_parsers::*, keypair::*},
+    solana_cli_config::ConfigInput,
     solana_cli_output::{
         display::println_name_value, CliSignature, CliValidatorsSortOrder, OutputFormat,
     },
@@ -162,6 +163,7 @@ pub enum CliCommand {
         address: Option<SignerIndex>,
         use_deprecated_loader: bool,
         allow_excessive_balance: bool,
+        skip_fee_check: bool,
     },
     Program(ProgramCliCommand),
     // Stake Commands
@@ -186,6 +188,7 @@ pub enum CliCommand {
         stake_account_pubkey: Pubkey,
         stake_authority: SignerIndex,
         sign_only: bool,
+        deactivate_delinquent: bool,
         dump_transaction_message: bool,
         blockhash_query: BlockhashQuery,
         nonce_account: Option<Pubkey>,
@@ -455,129 +458,23 @@ impl From<nonce_utils::Error> for CliError {
     }
 }
 
-pub enum SettingType {
-    Explicit,
-    Computed,
-    SystemDefault,
-}
-
 pub struct CliConfig<'a> {
     pub command: CliCommand,
     pub json_rpc_url: String,
     pub websocket_url: String,
-    pub signers: Vec<&'a dyn Signer>,
     pub keypair_path: String,
+    pub commitment: CommitmentConfig,
+    pub signers: Vec<&'a dyn Signer>,
     pub rpc_client: Option<Arc<RpcClient>>,
     pub rpc_timeout: Duration,
     pub verbose: bool,
     pub output_format: OutputFormat,
-    pub commitment: CommitmentConfig,
     pub send_transaction_config: RpcSendTransactionConfig,
     pub confirm_transaction_initial_timeout: Duration,
     pub address_labels: HashMap<String, String>,
 }
 
 impl CliConfig<'_> {
-    fn default_keypair_path() -> String {
-        solana_cli_config::Config::default().keypair_path
-    }
-
-    fn default_json_rpc_url() -> String {
-        solana_cli_config::Config::default().json_rpc_url
-    }
-
-    fn default_websocket_url() -> String {
-        solana_cli_config::Config::default().websocket_url
-    }
-
-    fn default_commitment() -> CommitmentConfig {
-        CommitmentConfig::confirmed()
-    }
-
-    fn first_nonempty_setting(
-        settings: std::vec::Vec<(SettingType, String)>,
-    ) -> (SettingType, String) {
-        settings
-            .into_iter()
-            .find(|(_, value)| !value.is_empty())
-            .expect("no nonempty setting")
-    }
-
-    fn first_setting_is_some<T>(
-        settings: std::vec::Vec<(SettingType, Option<T>)>,
-    ) -> (SettingType, T) {
-        let (setting_type, setting_option) = settings
-            .into_iter()
-            .find(|(_, value)| value.is_some())
-            .expect("all settings none");
-        (setting_type, setting_option.unwrap())
-    }
-
-    pub fn compute_websocket_url_setting(
-        websocket_cmd_url: &str,
-        websocket_cfg_url: &str,
-        json_rpc_cmd_url: &str,
-        json_rpc_cfg_url: &str,
-    ) -> (SettingType, String) {
-        Self::first_nonempty_setting(vec![
-            (SettingType::Explicit, websocket_cmd_url.to_string()),
-            (SettingType::Explicit, websocket_cfg_url.to_string()),
-            (
-                SettingType::Computed,
-                solana_cli_config::Config::compute_websocket_url(&normalize_to_url_if_moniker(
-                    json_rpc_cmd_url,
-                )),
-            ),
-            (
-                SettingType::Computed,
-                solana_cli_config::Config::compute_websocket_url(&normalize_to_url_if_moniker(
-                    json_rpc_cfg_url,
-                )),
-            ),
-            (SettingType::SystemDefault, Self::default_websocket_url()),
-        ])
-    }
-
-    pub fn compute_json_rpc_url_setting(
-        json_rpc_cmd_url: &str,
-        json_rpc_cfg_url: &str,
-    ) -> (SettingType, String) {
-        let (setting_type, url_or_moniker) = Self::first_nonempty_setting(vec![
-            (SettingType::Explicit, json_rpc_cmd_url.to_string()),
-            (SettingType::Explicit, json_rpc_cfg_url.to_string()),
-            (SettingType::SystemDefault, Self::default_json_rpc_url()),
-        ]);
-        (setting_type, normalize_to_url_if_moniker(&url_or_moniker))
-    }
-
-    pub fn compute_keypair_path_setting(
-        keypair_cmd_path: &str,
-        keypair_cfg_path: &str,
-    ) -> (SettingType, String) {
-        Self::first_nonempty_setting(vec![
-            (SettingType::Explicit, keypair_cmd_path.to_string()),
-            (SettingType::Explicit, keypair_cfg_path.to_string()),
-            (SettingType::SystemDefault, Self::default_keypair_path()),
-        ])
-    }
-
-    pub fn compute_commitment_config(
-        commitment_cmd: &str,
-        commitment_cfg: &str,
-    ) -> (SettingType, CommitmentConfig) {
-        Self::first_setting_is_some(vec![
-            (
-                SettingType::Explicit,
-                CommitmentConfig::from_str(commitment_cmd).ok(),
-            ),
-            (
-                SettingType::Explicit,
-                CommitmentConfig::from_str(commitment_cfg).ok(),
-            ),
-            (SettingType::SystemDefault, Some(Self::default_commitment())),
-        ])
-    }
-
     pub(crate) fn pubkey(&self) -> Result<Pubkey, SignerError> {
         if !self.signers.is_empty() {
             self.signers[0].try_pubkey()
@@ -608,15 +505,15 @@ impl Default for CliConfig<'_> {
                 pubkey: Some(Pubkey::default()),
                 use_lamports_unit: false,
             },
-            json_rpc_url: Self::default_json_rpc_url(),
-            websocket_url: Self::default_websocket_url(),
+            json_rpc_url: ConfigInput::default().json_rpc_url,
+            websocket_url: ConfigInput::default().websocket_url,
+            keypair_path: ConfigInput::default().keypair_path,
+            commitment: ConfigInput::default().commitment,
             signers: Vec::new(),
-            keypair_path: Self::default_keypair_path(),
             rpc_client: None,
             rpc_timeout: Duration::from_secs(u64::from_str(DEFAULT_RPC_TIMEOUT_SECONDS).unwrap()),
             verbose: false,
             output_format: OutputFormat::Display,
-            commitment: CommitmentConfig::confirmed(),
             send_transaction_config: RpcSendTransactionConfig::default(),
             confirm_transaction_initial_timeout: Duration::from_secs(
                 u64::from_str(DEFAULT_CONFIRM_TX_TIMEOUT_SECONDS).unwrap(),
@@ -744,6 +641,7 @@ pub fn parse_command(
                 signers.push(signer);
                 1
             });
+            let skip_fee_check = matches.is_present("skip_fee_check");
 
             Ok(CliCommandInfo {
                 command: CliCommand::Deploy {
@@ -751,6 +649,7 @@ pub fn parse_command(
                     address,
                     use_deprecated_loader: matches.is_present("use_deprecated_loader"),
                     allow_excessive_balance: matches.is_present("allow_excessive_balance"),
+                    skip_fee_check,
                 },
                 signers,
             })
@@ -1129,6 +1028,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             address,
             use_deprecated_loader,
             allow_excessive_balance,
+            skip_fee_check,
         } => process_deploy(
             rpc_client,
             config,
@@ -1136,6 +1036,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             *address,
             *use_deprecated_loader,
             *allow_excessive_balance,
+            *skip_fee_check,
         ),
         CliCommand::Program(program_subcommand) => {
             process_program_subcommand(rpc_client, config, program_subcommand)
@@ -1183,6 +1084,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             stake_account_pubkey,
             stake_authority,
             sign_only,
+            deactivate_delinquent,
             dump_transaction_message,
             blockhash_query,
             nonce_account,
@@ -1196,6 +1098,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             stake_account_pubkey,
             *stake_authority,
             *sign_only,
+            *deactivate_delinquent,
             *dump_transaction_message,
             blockhash_query,
             *nonce_account,
@@ -1967,6 +1870,7 @@ mod tests {
                     address: None,
                     use_deprecated_loader: false,
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 },
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -1989,6 +1893,7 @@ mod tests {
                     address: Some(1),
                     use_deprecated_loader: false,
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 },
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -2190,6 +2095,7 @@ mod tests {
             stake_authority: 0,
             sign_only: false,
             dump_transaction_message: false,
+            deactivate_delinquent: false,
             blockhash_query: BlockhashQuery::default(),
             nonce_account: None,
             nonce_authority: 0,
@@ -2382,6 +2288,7 @@ mod tests {
             address: None,
             use_deprecated_loader: false,
             allow_excessive_balance: false,
+            skip_fee_check: false,
         };
         config.output_format = OutputFormat::JsonCompact;
         let result = process_command(&config);
@@ -2402,6 +2309,7 @@ mod tests {
             address: None,
             use_deprecated_loader: false,
             allow_excessive_balance: false,
+            skip_fee_check: false,
         };
         assert!(process_command(&config).is_err());
     }

cli/src/cluster_query.rs

@@ -33,7 +33,7 @@ use {
         rpc_request::DELINQUENT_VALIDATOR_SLOT_DISTANCE,
         rpc_response::SlotInfo,
     },
-    solana_program_runtime::compute_budget::ComputeBudget,
+    solana_program_runtime::compute_budget,
     solana_remote_wallet::remote_wallet::RemoteWalletManager,
     solana_sdk::{
         account::from_account,
@@ -379,6 +379,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                             "root",
                             "skip-rate",
                             "stake",
+                            "version",
                             "vote-account",
                         ])
                         .default_value("stake")
@@ -650,6 +651,7 @@ pub fn parse_show_validators(matches: &ArgMatches<'_>) -> Result<CliCommandInfo,
         "skip-rate" => CliValidatorsSortOrder::SkipRate,
         "stake" => CliValidatorsSortOrder::Stake,
         "vote-account" => CliValidatorsSortOrder::VoteAccount,
+        "version" => CliValidatorsSortOrder::Version,
         _ => unreachable!(),
     };
 
@@ -1409,7 +1411,7 @@ pub fn process_ping(
             )];
             if let Some(additional_fee) = additional_fee {
                 ixs.push(ComputeBudgetInstruction::request_units(
-                    ComputeBudget::new(false).max_units as u32,
+                    compute_budget::DEFAULT_UNITS,
                     *additional_fee,
                 ));
             }

cli/src/main.rs

@@ -8,30 +8,18 @@ use {
     },
     solana_cli::{
         clap_app::get_clap_app,
-        cli::{parse_command, process_command, CliCommandInfo, CliConfig, SettingType},
+        cli::{parse_command, process_command, CliCommandInfo, CliConfig},
+    },
+    solana_cli_config::{Config, ConfigInput},
+    solana_cli_output::{
+        display::{println_name_value, println_name_value_or},
+        OutputFormat,
     },
-    solana_cli_config::Config,
-    solana_cli_output::{display::println_name_value, OutputFormat},
     solana_client::rpc_config::RpcSendTransactionConfig,
     solana_remote_wallet::remote_wallet::RemoteWalletManager,
     std::{collections::HashMap, error, path::PathBuf, sync::Arc, time::Duration},
 };
 
-pub fn println_name_value_or(name: &str, value: &str, setting_type: SettingType) {
-    let description = match setting_type {
-        SettingType::Explicit => "",
-        SettingType::Computed => "(computed)",
-        SettingType::SystemDefault => "(default)",
-    };
-
-    println!(
-        "{} {} {}",
-        style(name).bold(),
-        style(value),
-        style(description).italic(),
-    );
-}
-
 fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error>> {
     let parse_args = match matches.subcommand() {
         ("config", Some(matches)) => {
@@ -50,17 +38,18 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
             match matches.subcommand() {
                 ("get", Some(subcommand_matches)) => {
                     let (url_setting_type, json_rpc_url) =
-                        CliConfig::compute_json_rpc_url_setting("", &config.json_rpc_url);
-                    let (ws_setting_type, websocket_url) = CliConfig::compute_websocket_url_setting(
+                        ConfigInput::compute_json_rpc_url_setting("", &config.json_rpc_url);
+                    let (ws_setting_type, websocket_url) =
+                        ConfigInput::compute_websocket_url_setting(
                             "",
                             &config.websocket_url,
                             "",
                             &config.json_rpc_url,
                         );
                     let (keypair_setting_type, keypair_path) =
-                        CliConfig::compute_keypair_path_setting("", &config.keypair_path);
+                        ConfigInput::compute_keypair_path_setting("", &config.keypair_path);
                     let (commitment_setting_type, commitment) =
-                        CliConfig::compute_commitment_config("", &config.commitment);
+                        ConfigInput::compute_commitment_config("", &config.commitment);
 
                     if let Some(field) = subcommand_matches.value_of("specific_setting") {
                         let (field_name, value, setting_type) = match field {
@@ -107,17 +96,18 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
                     config.save(config_file)?;
 
                     let (url_setting_type, json_rpc_url) =
-                        CliConfig::compute_json_rpc_url_setting("", &config.json_rpc_url);
-                    let (ws_setting_type, websocket_url) = CliConfig::compute_websocket_url_setting(
+                        ConfigInput::compute_json_rpc_url_setting("", &config.json_rpc_url);
+                    let (ws_setting_type, websocket_url) =
+                        ConfigInput::compute_websocket_url_setting(
                             "",
                             &config.websocket_url,
                             "",
                             &config.json_rpc_url,
                         );
                     let (keypair_setting_type, keypair_path) =
-                        CliConfig::compute_keypair_path_setting("", &config.keypair_path);
+                        ConfigInput::compute_keypair_path_setting("", &config.keypair_path);
                     let (commitment_setting_type, commitment) =
-                        CliConfig::compute_commitment_config("", &config.commitment);
+                        ConfigInput::compute_commitment_config("", &config.commitment);
 
                     println_name_value("Config File:", config_file);
                     println_name_value_or("RPC URL:", &json_rpc_url, url_setting_type);
@@ -158,7 +148,7 @@ pub fn parse_args<'a>(
     } else {
         Config::default()
     };
-    let (_, json_rpc_url) = CliConfig::compute_json_rpc_url_setting(
+    let (_, json_rpc_url) = ConfigInput::compute_json_rpc_url_setting(
         matches.value_of("json_rpc_url").unwrap_or(""),
         &config.json_rpc_url,
     );
@@ -171,14 +161,14 @@ pub fn parse_args<'a>(
     let confirm_transaction_initial_timeout =
         Duration::from_secs(confirm_transaction_initial_timeout);
 
-    let (_, websocket_url) = CliConfig::compute_websocket_url_setting(
+    let (_, websocket_url) = ConfigInput::compute_websocket_url_setting(
         matches.value_of("websocket_url").unwrap_or(""),
         &config.websocket_url,
         matches.value_of("json_rpc_url").unwrap_or(""),
         &config.json_rpc_url,
     );
     let default_signer_arg_name = "keypair".to_string();
-    let (_, default_signer_path) = CliConfig::compute_keypair_path_setting(
+    let (_, default_signer_path) = ConfigInput::compute_keypair_path_setting(
         matches.value_of(&default_signer_arg_name).unwrap_or(""),
         &config.keypair_path,
     );
@@ -201,7 +191,7 @@ pub fn parse_args<'a>(
     let verbose = matches.is_present("verbose");
     let output_format = OutputFormat::from_matches(matches, "output_format", verbose);
 
-    let (_, commitment) = CliConfig::compute_commitment_config(
+    let (_, commitment) = ConfigInput::compute_commitment_config(
         matches.value_of("commitment").unwrap_or(""),
         &config.commitment,
     );

cli/src/program.rs

@@ -66,6 +66,7 @@ pub enum ProgramCliCommand {
         is_final: bool,
         max_len: Option<usize>,
         allow_excessive_balance: bool,
+        skip_fee_check: bool,
     },
     WriteBuffer {
         program_location: String,
@@ -73,6 +74,7 @@ pub enum ProgramCliCommand {
         buffer_pubkey: Option<Pubkey>,
         buffer_authority_signer_index: Option<SignerIndex>,
         max_len: Option<usize>,
+        skip_fee_check: bool,
     },
     SetBufferAuthority {
         buffer_pubkey: Pubkey,
@@ -114,6 +116,13 @@ impl ProgramSubCommands for App<'_, '_> {
             SubCommand::with_name("program")
                 .about("Program management")
                 .setting(AppSettings::SubcommandRequiredElseHelp)
+                .arg(
+                    Arg::with_name("skip_fee_check")
+                        .long("skip-fee-check")
+                        .hidden(true)
+                        .takes_value(false)
+                        .global(true)
+                )
                 .subcommand(
                     SubCommand::with_name("deploy")
                         .about("Deploy a program")
@@ -406,6 +415,12 @@ impl ProgramSubCommands for App<'_, '_> {
                         .long("allow-excessive-deploy-account-balance")
                         .takes_value(false)
                         .help("Use the designated program id, even if the account already holds a large balance of SOL")
+                )
+                .arg(
+                    Arg::with_name("skip_fee_check")
+                        .long("skip-fee-check")
+                        .hidden(true)
+                        .takes_value(false)
                 ),
         )
     }
@@ -416,7 +431,14 @@ pub fn parse_program_subcommand(
     default_signer: &DefaultSigner,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let response = match matches.subcommand() {
+    let (subcommand, sub_matches) = matches.subcommand();
+    let matches_skip_fee_check = matches.is_present("skip_fee_check");
+    let sub_matches_skip_fee_check = sub_matches
+        .map(|m| m.is_present("skip_fee_check"))
+        .unwrap_or(false);
+    let skip_fee_check = matches_skip_fee_check || sub_matches_skip_fee_check;
+
+    let response = match (subcommand, sub_matches) {
         ("deploy", Some(matches)) => {
             let mut bulk_signers = vec![Some(
                 default_signer.signer_from_path(matches, wallet_manager)?,
@@ -476,6 +498,7 @@ pub fn parse_program_subcommand(
                     is_final: matches.is_present("final"),
                     max_len,
                     allow_excessive_balance: matches.is_present("allow_excessive_balance"),
+                    skip_fee_check,
                 }),
                 signers: signer_info.signers,
             }
@@ -521,6 +544,7 @@ pub fn parse_program_subcommand(
                     buffer_authority_signer_index: signer_info
                         .index_of_or_none(buffer_authority_pubkey),
                     max_len,
+                    skip_fee_check,
                 }),
                 signers: signer_info.signers,
             }
@@ -669,6 +693,7 @@ pub fn process_program_subcommand(
             is_final,
             max_len,
             allow_excessive_balance,
+            skip_fee_check,
         } => process_program_deploy(
             rpc_client,
             config,
@@ -681,6 +706,7 @@ pub fn process_program_subcommand(
             *is_final,
             *max_len,
             *allow_excessive_balance,
+            *skip_fee_check,
         ),
         ProgramCliCommand::WriteBuffer {
             program_location,
@@ -688,6 +714,7 @@ pub fn process_program_subcommand(
             buffer_pubkey,
             buffer_authority_signer_index,
             max_len,
+            skip_fee_check,
         } => process_write_buffer(
             rpc_client,
             config,
@@ -696,6 +723,7 @@ pub fn process_program_subcommand(
             *buffer_pubkey,
             *buffer_authority_signer_index,
             *max_len,
+            *skip_fee_check,
         ),
         ProgramCliCommand::SetBufferAuthority {
             buffer_pubkey,
@@ -793,6 +821,7 @@ fn process_program_deploy(
     is_final: bool,
     max_len: Option<usize>,
     allow_excessive_balance: bool,
+    skip_fee_check: bool,
 ) -> ProcessResult {
     let (words, mnemonic, buffer_keypair) = create_ephemeral_keypair()?;
     let (buffer_provided, buffer_signer, buffer_pubkey) = if let Some(i) = buffer_signer_index {
@@ -947,6 +976,7 @@ fn process_program_deploy(
             &buffer_pubkey,
             Some(upgrade_authority_signer),
             allow_excessive_balance,
+            skip_fee_check,
         )
     } else {
         do_process_program_upgrade(
@@ -957,6 +987,7 @@ fn process_program_deploy(
             config.signers[upgrade_authority_signer_index],
             &buffer_pubkey,
             buffer_signer,
+            skip_fee_check,
         )
     };
     if result.is_ok() && is_final {
@@ -983,6 +1014,7 @@ fn process_write_buffer(
     buffer_pubkey: Option<Pubkey>,
     buffer_authority_signer_index: Option<SignerIndex>,
     max_len: Option<usize>,
+    skip_fee_check: bool,
 ) -> ProcessResult {
     // Create ephemeral keypair to use for Buffer account, if not provided
     let (words, mnemonic, buffer_keypair) = create_ephemeral_keypair()?;
@@ -1050,6 +1082,7 @@ fn process_write_buffer(
         &buffer_pubkey,
         Some(buffer_authority),
         true,
+        skip_fee_check,
     );
 
     if result.is_err() && buffer_signer_index.is_none() && buffer_signer.is_some() {
@@ -1636,6 +1669,7 @@ pub fn process_deploy(
     buffer_signer_index: Option<SignerIndex>,
     use_deprecated_loader: bool,
     allow_excessive_balance: bool,
+    skip_fee_check: bool,
 ) -> ProcessResult {
     // Create ephemeral keypair to use for Buffer account, if not provided
     let (words, mnemonic, buffer_keypair) = create_ephemeral_keypair()?;
@@ -1666,6 +1700,7 @@ pub fn process_deploy(
         &buffer_signer.pubkey(),
         Some(buffer_signer),
         allow_excessive_balance,
+        skip_fee_check,
     );
     if result.is_err() && buffer_signer_index.is_none() {
         report_ephemeral_mnemonic(words, mnemonic);
@@ -1704,6 +1739,7 @@ fn do_process_program_write_and_deploy(
     buffer_pubkey: &Pubkey,
     buffer_authority_signer: Option<&dyn Signer>,
     allow_excessive_balance: bool,
+    skip_fee_check: bool,
 ) -> ProcessResult {
     // Build messages to calculate fees
     let mut messages: Vec<&Message> = Vec::new();
@@ -1834,7 +1870,9 @@ fn do_process_program_write_and_deploy(
         messages.push(message);
     }
 
+    if !skip_fee_check {
         check_payer(&rpc_client, config, balance_needed, &messages)?;
+    }
 
     send_deploy_messages(
         rpc_client,
@@ -1868,6 +1906,7 @@ fn do_process_program_upgrade(
     upgrade_authority: &dyn Signer,
     buffer_pubkey: &Pubkey,
     buffer_signer: Option<&dyn Signer>,
+    skip_fee_check: bool,
 ) -> ProcessResult {
     let loader_id = bpf_loader_upgradeable::id();
     let data_len = program_data.len();
@@ -1967,7 +2006,10 @@ fn do_process_program_upgrade(
     );
     messages.push(&final_message);
 
+    if !skip_fee_check {
         check_payer(&rpc_client, config, balance_needed, &messages)?;
+    }
+
     send_deploy_messages(
         rpc_client,
         config,
@@ -2255,6 +2297,7 @@ mod tests {
                     is_final: false,
                     max_len: None,
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 }),
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -2281,6 +2324,7 @@ mod tests {
                     is_final: false,
                     max_len: Some(42),
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 }),
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -2309,6 +2353,7 @@ mod tests {
                     is_final: false,
                     max_len: None,
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 }),
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -2339,6 +2384,7 @@ mod tests {
                     is_final: false,
                     max_len: None,
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 }),
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -2368,6 +2414,7 @@ mod tests {
                     is_final: false,
                     max_len: None,
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 }),
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -2400,6 +2447,7 @@ mod tests {
                     is_final: false,
                     max_len: None,
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 }),
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -2427,6 +2475,7 @@ mod tests {
                     upgrade_authority_signer_index: 0,
                     is_final: true,
                     max_len: None,
+                    skip_fee_check: false,
                     allow_excessive_balance: false,
                 }),
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
@@ -2460,6 +2509,7 @@ mod tests {
                     buffer_pubkey: None,
                     buffer_authority_signer_index: Some(0),
                     max_len: None,
+                    skip_fee_check: false,
                 }),
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -2483,6 +2533,7 @@ mod tests {
                     buffer_pubkey: None,
                     buffer_authority_signer_index: Some(0),
                     max_len: Some(42),
+                    skip_fee_check: false,
                 }),
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -2509,6 +2560,7 @@ mod tests {
                     buffer_pubkey: Some(buffer_keypair.pubkey()),
                     buffer_authority_signer_index: Some(0),
                     max_len: None,
+                    skip_fee_check: false,
                 }),
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -2538,6 +2590,7 @@ mod tests {
                     buffer_pubkey: None,
                     buffer_authority_signer_index: Some(1),
                     max_len: None,
+                    skip_fee_check: false,
                 }),
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -2572,6 +2625,7 @@ mod tests {
                     buffer_pubkey: Some(buffer_keypair.pubkey()),
                     buffer_authority_signer_index: Some(2),
                     max_len: None,
+                    skip_fee_check: false,
                 }),
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -3014,6 +3068,7 @@ mod tests {
                 is_final: false,
                 max_len: None,
                 allow_excessive_balance: false,
+                skip_fee_check: false,
             }),
             signers: vec![&default_keypair],
             output_format: OutputFormat::JsonCompact,

cli/src/stake.rs

@@ -41,6 +41,7 @@ use {
             self,
             instruction::{self as stake_instruction, LockupArgs, StakeError},
             state::{Authorized, Lockup, Meta, StakeActivationStatus, StakeAuthorize, StakeState},
+            tools::{acceptable_reference_epoch_credits, eligible_for_deactivate_delinquent},
         },
         stake_history::StakeHistory,
         system_instruction::SystemError,
@@ -379,6 +380,13 @@ impl StakeSubCommands for App<'_, '_> {
                         .help("Seed for address generation; if specified, the resulting account \
                                will be at a derived address of STAKE_ACCOUNT_ADDRESS")
                 )
+                .arg(
+                    Arg::with_name("delinquent")
+                        .long("delinquent")
+                        .takes_value(false)
+                        .conflicts_with(SIGN_ONLY_ARG.name)
+                        .help("Deactivate abandoned stake that is currently delegated to a delinquent vote account")
+                )
                 .arg(stake_authority_arg())
                 .offline_args()
                 .nonce_args(false)
@@ -995,11 +1003,13 @@ pub fn parse_stake_deactivate_stake(
     let stake_account_pubkey =
         pubkey_of_signer(matches, "stake_account_pubkey", wallet_manager)?.unwrap();
     let sign_only = matches.is_present(SIGN_ONLY_ARG.name);
+    let deactivate_delinquent = matches.is_present("delinquent");
     let dump_transaction_message = matches.is_present(DUMP_TRANSACTION_MESSAGE.name);
     let blockhash_query = BlockhashQuery::new_from_matches(matches);
     let nonce_account = pubkey_of(matches, NONCE_ARG.name);
     let memo = matches.value_of(MEMO_ARG.name).map(String::from);
     let seed = value_t!(matches, "seed", String).ok();
+
     let (stake_authority, stake_authority_pubkey) =
         signer_of(matches, STAKE_AUTHORITY_ARG.name, wallet_manager)?;
     let (nonce_authority, nonce_authority_pubkey) =
@@ -1018,6 +1028,7 @@ pub fn parse_stake_deactivate_stake(
             stake_account_pubkey,
             stake_authority: signer_info.index_of(stake_authority_pubkey).unwrap(),
             sign_only,
+            deactivate_delinquent,
             dump_transaction_message,
             blockhash_query,
             nonce_account,
@@ -1383,17 +1394,12 @@ pub fn process_stake_authorize(
             };
             if let Some(authorized) = authorized {
                 match authorization_type {
-                    StakeAuthorize::Staker => {
-                        // first check authorized withdrawer
-                        check_current_authority(&authorized.withdrawer, &authority.pubkey())
-                            .or_else(|_| {
-                                // ...then check authorized staker. If neither matches, error will
-                                // print the stake key as `expected`
-                                check_current_authority(&authorized.staker, &authority.pubkey())
-                            })?;
-                    }
+                    StakeAuthorize::Staker => check_current_authority(
+                        &[authorized.withdrawer, authorized.staker],
+                        &authority.pubkey(),
+                    )?,
                     StakeAuthorize::Withdrawer => {
-                        check_current_authority(&authorized.withdrawer, &authority.pubkey())?;
+                        check_current_authority(&[authorized.withdrawer], &authority.pubkey())?;
                     }
                 }
             } else {
@@ -1482,6 +1488,7 @@ pub fn process_deactivate_stake_account(
     stake_account_pubkey: &Pubkey,
     stake_authority: SignerIndex,
     sign_only: bool,
+    deactivate_delinquent: bool,
     dump_transaction_message: bool,
     blockhash_query: &BlockhashQuery,
     nonce_account: Option<Pubkey>,
@@ -1491,7 +1498,6 @@ pub fn process_deactivate_stake_account(
     fee_payer: SignerIndex,
 ) -> ProcessResult {
     let recent_blockhash = blockhash_query.get_blockhash(rpc_client, config.commitment)?;
-    let stake_authority = config.signers[stake_authority];
 
     let stake_account_address = if let Some(seed) = seed {
         Pubkey::create_with_seed(stake_account_pubkey, seed, &stake::program::id())?
@@ -1499,11 +1505,77 @@ pub fn process_deactivate_stake_account(
         *stake_account_pubkey
     };
 
-    let ixs = vec![stake_instruction::deactivate_stake(
+    let ixs = vec![if deactivate_delinquent {
+        let stake_account = rpc_client.get_account(&stake_account_address)?;
+        if stake_account.owner != stake::program::id() {
+            return Err(CliError::BadParameter(format!(
+                "{} is not a stake account",
+                stake_account_address,
+            ))
+            .into());
+        }
+
+        let vote_account_address = match stake_account.state() {
+            Ok(stake_state) => match stake_state {
+                StakeState::Stake(_, stake) => stake.delegation.voter_pubkey,
+                _ => {
+                    return Err(CliError::BadParameter(format!(
+                        "{} is not a delegated stake account",
+                        stake_account_address,
+                    ))
+                    .into())
+                }
+            },
+            Err(err) => {
+                return Err(CliError::RpcRequestError(format!(
+                    "Account data could not be deserialized to stake state: {}",
+                    err
+                ))
+                .into())
+            }
+        };
+
+        let current_epoch = rpc_client.get_epoch_info()?.epoch;
+
+        let (_, vote_state) = crate::vote::get_vote_account(
+            rpc_client,
+            &vote_account_address,
+            rpc_client.commitment(),
+        )?;
+        if !eligible_for_deactivate_delinquent(&vote_state.epoch_credits, current_epoch) {
+            return Err(CliError::BadParameter(format!(
+                "Stake has not been delinquent for {} epochs",
+                stake::MINIMUM_DELINQUENT_EPOCHS_FOR_DEACTIVATION,
+            ))
+            .into());
+        }
+
+        // Search for a reference vote account
+        let reference_vote_account_address = rpc_client
+            .get_vote_accounts()?
+            .current
+            .into_iter()
+            .find(|vote_account_info| {
+                acceptable_reference_epoch_credits(&vote_account_info.epoch_credits, current_epoch)
+            });
+        let reference_vote_account_address = reference_vote_account_address
+            .ok_or_else(|| {
+                CliError::RpcRequestError("Unable to find a reference vote account".into())
+            })?
+            .vote_pubkey
+            .parse()?;
+
+        stake_instruction::deactivate_delinquent_stake(
             &stake_account_address,
-        &stake_authority.pubkey(),
-    )]
+            &vote_account_address,
+            &reference_vote_account_address,
+        )
+    } else {
+        let stake_authority = config.signers[stake_authority];
+        stake_instruction::deactivate_stake(&stake_account_address, &stake_authority.pubkey())
+    }]
     .with_memo(memo);
+
     let nonce_authority = config.signers[nonce_authority];
     let fee_payer = config.signers[fee_payer];
 
@@ -1935,7 +2007,7 @@ pub fn process_stake_set_lockup(
         };
         if let Some(lockup) = lockup {
             if lockup.custodian != Pubkey::default() {
-                check_current_authority(&lockup.custodian, &custodian.pubkey())?;
+                check_current_authority(&[lockup.custodian], &custodian.pubkey())?;
             }
         } else {
             return Err(CliError::RpcRequestError(format!(
@@ -2119,13 +2191,13 @@ fn get_stake_account_state(
 }
 
 pub(crate) fn check_current_authority(
-    account_current_authority: &Pubkey,
+    permitted_authorities: &[Pubkey],
     provided_current_authority: &Pubkey,
 ) -> Result<(), CliError> {
-    if account_current_authority != provided_current_authority {
+    if !permitted_authorities.contains(provided_current_authority) {
         Err(CliError::RpcRequestError(format!(
-            "Invalid current authority provided: {:?}, expected {:?}",
-            provided_current_authority, account_current_authority
+            "Invalid authority provided: {:?}, expected {:?}",
+            provided_current_authority, permitted_authorities
         )))
     } else {
         Ok(())
@@ -4179,6 +4251,34 @@ mod tests {
                     stake_account_pubkey,
                     stake_authority: 0,
                     sign_only: false,
+                    deactivate_delinquent: false,
+                    dump_transaction_message: false,
+                    blockhash_query: BlockhashQuery::default(),
+                    nonce_account: None,
+                    nonce_authority: 0,
+                    memo: None,
+                    seed: None,
+                    fee_payer: 0,
+                },
+                signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
+            }
+        );
+
+        // Test DeactivateStake Subcommand with delinquent flag
+        let test_deactivate_stake = test_commands.clone().get_matches_from(vec![
+            "test",
+            "deactivate-stake",
+            &stake_account_string,
+            "--delinquent",
+        ]);
+        assert_eq!(
+            parse_command(&test_deactivate_stake, &default_signer, &mut None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::DeactivateStake {
+                    stake_account_pubkey,
+                    stake_authority: 0,
+                    sign_only: false,
+                    deactivate_delinquent: true,
                     dump_transaction_message: false,
                     blockhash_query: BlockhashQuery::default(),
                     nonce_account: None,
@@ -4206,6 +4306,7 @@ mod tests {
                     stake_account_pubkey,
                     stake_authority: 1,
                     sign_only: false,
+                    deactivate_delinquent: false,
                     dump_transaction_message: false,
                     blockhash_query: BlockhashQuery::default(),
                     nonce_account: None,
@@ -4240,6 +4341,7 @@ mod tests {
                     stake_account_pubkey,
                     stake_authority: 0,
                     sign_only: false,
+                    deactivate_delinquent: false,
                     dump_transaction_message: false,
                     blockhash_query: BlockhashQuery::FeeCalculator(
                         blockhash_query::Source::Cluster,
@@ -4270,6 +4372,7 @@ mod tests {
                     stake_account_pubkey,
                     stake_authority: 0,
                     sign_only: true,
+                    deactivate_delinquent: false,
                     dump_transaction_message: false,
                     blockhash_query: BlockhashQuery::None(blockhash),
                     nonce_account: None,
@@ -4304,6 +4407,7 @@ mod tests {
                     stake_account_pubkey,
                     stake_authority: 0,
                     sign_only: false,
+                    deactivate_delinquent: false,
                     dump_transaction_message: false,
                     blockhash_query: BlockhashQuery::FeeCalculator(
                         blockhash_query::Source::Cluster,
@@ -4350,6 +4454,7 @@ mod tests {
                     stake_account_pubkey,
                     stake_authority: 0,
                     sign_only: false,
+                    deactivate_delinquent: false,
                     dump_transaction_message: false,
                     blockhash_query: BlockhashQuery::FeeCalculator(
                         blockhash_query::Source::NonceAccount(nonce_account),
@@ -4384,6 +4489,7 @@ mod tests {
                     stake_account_pubkey,
                     stake_authority: 0,
                     sign_only: false,
+                    deactivate_delinquent: false,
                     dump_transaction_message: false,
                     blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
                     nonce_account: None,

cli/src/vote.rs

@@ -910,7 +910,10 @@ pub fn process_vote_authorize(
                             "Invalid vote account state; no authorized voters found".to_string(),
                         )
                     })?;
-                check_current_authority(&current_authorized_voter, &authorized.pubkey())?;
+                check_current_authority(
+                    &[current_authorized_voter, vote_state.authorized_withdrawer],
+                    &authorized.pubkey(),
+                )?;
                 if let Some(signer) = new_authorized_signer {
                     if signer.is_interactive() {
                         return Err(CliError::BadParameter(format!(
@@ -927,7 +930,7 @@ pub fn process_vote_authorize(
                 (new_authorized_pubkey, "new_authorized_pubkey".to_string()),
             )?;
             if let Some(vote_state) = vote_state {
-                check_current_authority(&vote_state.authorized_withdrawer, &authorized.pubkey())?
+                check_current_authority(&[vote_state.authorized_withdrawer], &authorized.pubkey())?
             }
         }
     }
@@ -1137,7 +1140,7 @@ pub fn process_vote_update_commission(
     }
 }
 
-fn get_vote_account(
+pub(crate) fn get_vote_account(
     rpc_client: &RpcClient,
     vote_account_pubkey: &Pubkey,
     commitment_config: CommitmentConfig,

cli/tests/program.rs

@@ -62,6 +62,7 @@ fn test_cli_program_deploy_non_upgradeable() {
         address: None,
         use_deprecated_loader: false,
         allow_excessive_balance: false,
+        skip_fee_check: false,
     };
     config.output_format = OutputFormat::JsonCompact;
     let response = process_command(&config);
@@ -91,6 +92,7 @@ fn test_cli_program_deploy_non_upgradeable() {
         address: Some(1),
         use_deprecated_loader: false,
         allow_excessive_balance: false,
+        skip_fee_check: false,
     };
     process_command(&config).unwrap();
     let account1 = rpc_client
@@ -118,6 +120,7 @@ fn test_cli_program_deploy_non_upgradeable() {
         address: Some(1),
         use_deprecated_loader: false,
         allow_excessive_balance: false,
+        skip_fee_check: false,
     };
     process_command(&config).unwrap_err();
 
@@ -127,6 +130,7 @@ fn test_cli_program_deploy_non_upgradeable() {
         address: Some(1),
         use_deprecated_loader: false,
         allow_excessive_balance: true,
+        skip_fee_check: false,
     };
     process_command(&config).unwrap();
     let account2 = rpc_client
@@ -193,6 +197,7 @@ fn test_cli_program_deploy_no_authority() {
         upgrade_authority_signer_index: 1,
         is_final: true,
         max_len: None,
+        skip_fee_check: false,
     });
     config.output_format = OutputFormat::JsonCompact;
     let response = process_command(&config);
@@ -218,6 +223,7 @@ fn test_cli_program_deploy_no_authority() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap_err();
 }
@@ -278,6 +284,7 @@ fn test_cli_program_deploy_with_authority() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: Some(max_len),
+        skip_fee_check: false,
     });
     config.output_format = OutputFormat::JsonCompact;
     let response = process_command(&config);
@@ -325,6 +332,7 @@ fn test_cli_program_deploy_with_authority() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: Some(max_len),
+        skip_fee_check: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -366,6 +374,7 @@ fn test_cli_program_deploy_with_authority() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: Some(max_len),
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
     let program_account = rpc_client.get_account(&program_pubkey).unwrap();
@@ -420,6 +429,7 @@ fn test_cli_program_deploy_with_authority() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
     let program_account = rpc_client.get_account(&program_pubkey).unwrap();
@@ -494,6 +504,7 @@ fn test_cli_program_deploy_with_authority() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap_err();
 
@@ -509,6 +520,7 @@ fn test_cli_program_deploy_with_authority() {
         upgrade_authority_signer_index: 1,
         is_final: true,
         max_len: None,
+        skip_fee_check: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -611,6 +623,7 @@ fn test_cli_program_close_program() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: Some(max_len),
+        skip_fee_check: false,
     });
     config.output_format = OutputFormat::JsonCompact;
     process_command(&config).unwrap();
@@ -695,6 +708,7 @@ fn test_cli_program_write_buffer() {
         buffer_pubkey: None,
         buffer_authority_signer_index: None,
         max_len: None,
+        skip_fee_check: false,
     });
     config.output_format = OutputFormat::JsonCompact;
     let response = process_command(&config);
@@ -729,6 +743,7 @@ fn test_cli_program_write_buffer() {
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: None,
         max_len: Some(max_len),
+        skip_fee_check: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -790,6 +805,7 @@ fn test_cli_program_write_buffer() {
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: Some(2),
         max_len: None,
+        skip_fee_check: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -827,6 +843,7 @@ fn test_cli_program_write_buffer() {
         buffer_pubkey: None,
         buffer_authority_signer_index: Some(2),
         max_len: None,
+        skip_fee_check: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -899,6 +916,7 @@ fn test_cli_program_write_buffer() {
         buffer_pubkey: None,
         buffer_authority_signer_index: None,
         max_len: None,
+        skip_fee_check: false,
     });
     config.output_format = OutputFormat::JsonCompact;
     let response = process_command(&config);
@@ -938,6 +956,7 @@ fn test_cli_program_write_buffer() {
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: None,
         max_len: None, //Some(max_len),
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
     config.signers = vec![&keypair, &buffer_keypair];
@@ -951,6 +970,7 @@ fn test_cli_program_write_buffer() {
         upgrade_authority_signer_index: 1,
         is_final: true,
         max_len: None,
+        skip_fee_check: false,
     });
     config.output_format = OutputFormat::JsonCompact;
     let error = process_command(&config).unwrap_err();
@@ -1008,6 +1028,7 @@ fn test_cli_program_set_buffer_authority() {
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: None,
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
     let buffer_account = rpc_client.get_account(&buffer_keypair.pubkey()).unwrap();
@@ -1123,6 +1144,7 @@ fn test_cli_program_mismatch_buffer_authority() {
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: Some(2),
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
     let buffer_account = rpc_client.get_account(&buffer_keypair.pubkey()).unwrap();
@@ -1145,6 +1167,7 @@ fn test_cli_program_mismatch_buffer_authority() {
         upgrade_authority_signer_index: 1,
         is_final: true,
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap_err();
 
@@ -1160,6 +1183,7 @@ fn test_cli_program_mismatch_buffer_authority() {
         upgrade_authority_signer_index: 1,
         is_final: true,
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
 }
@@ -1216,6 +1240,7 @@ fn test_cli_program_show() {
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: Some(2),
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
 
@@ -1275,6 +1300,7 @@ fn test_cli_program_show() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: Some(max_len),
+        skip_fee_check: false,
     });
     config.output_format = OutputFormat::JsonCompact;
     let min_slot = rpc_client.get_slot().unwrap();
@@ -1401,6 +1427,7 @@ fn test_cli_program_dump() {
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: Some(2),
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
 

cli/tests/stake.rs

@@ -204,6 +204,7 @@ fn test_seed_stake_delegation_and_deactivation() {
         stake_account_pubkey: stake_address,
         stake_authority: 0,
         sign_only: false,
+        deactivate_delinquent: false,
         dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
@@ -287,6 +288,7 @@ fn test_stake_delegation_and_deactivation() {
         stake_account_pubkey: stake_keypair.pubkey(),
         stake_authority: 0,
         sign_only: false,
+        deactivate_delinquent: false,
         dump_transaction_message: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
@@ -412,6 +414,7 @@ fn test_offline_stake_delegation_and_deactivation() {
         stake_account_pubkey: stake_keypair.pubkey(),
         stake_authority: 0,
         sign_only: true,
+        deactivate_delinquent: false,
         dump_transaction_message: false,
         blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
@@ -431,6 +434,7 @@ fn test_offline_stake_delegation_and_deactivation() {
         stake_account_pubkey: stake_keypair.pubkey(),
         stake_authority: 0,
         sign_only: false,
+        deactivate_delinquent: false,
         dump_transaction_message: false,
         blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
@@ -546,6 +550,7 @@ fn test_nonced_stake_delegation_and_deactivation() {
         stake_account_pubkey: stake_keypair.pubkey(),
         stake_authority: 0,
         sign_only: false,
+        deactivate_delinquent: false,
         dump_transaction_message: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account.pubkey()),

client-test/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client-test"
-version = "1.10.3"
+version = "1.11.0"
 description = "Solana RPC Test"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,25 +14,25 @@ publish = false
 futures-util = "0.3.21"
 serde_json = "1.0.79"
 serial_test = "0.6.0"
-solana-client = { path = "../client", version = "=1.10.3" }
-solana-ledger = { path = "../ledger", version = "=1.10.3" }
-solana-measure = { path = "../measure", version = "=1.10.3" }
-solana-merkle-tree = { path = "../merkle-tree", version = "=1.10.3" }
-solana-metrics = { path = "../metrics", version = "=1.10.3" }
-solana-perf = { path = "../perf", version = "=1.10.3" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.10.3" }
-solana-rpc = { path = "../rpc", version = "=1.10.3" }
-solana-runtime = { path = "../runtime", version = "=1.10.3" }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
-solana-streamer = { path = "../streamer", version = "=1.10.3" }
-solana-test-validator = { path = "../test-validator", version = "=1.10.3" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.3" }
-solana-version = { path = "../version", version = "=1.10.3" }
+solana-client = { path = "../client", version = "=1.11.0" }
+solana-ledger = { path = "../ledger", version = "=1.11.0" }
+solana-measure = { path = "../measure", version = "=1.11.0" }
+solana-merkle-tree = { path = "../merkle-tree", version = "=1.11.0" }
+solana-metrics = { path = "../metrics", version = "=1.11.0" }
+solana-perf = { path = "../perf", version = "=1.11.0" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.11.0" }
+solana-rpc = { path = "../rpc", version = "=1.11.0" }
+solana-runtime = { path = "../runtime", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
+solana-streamer = { path = "../streamer", version = "=1.11.0" }
+solana-test-validator = { path = "../test-validator", version = "=1.11.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.11.0" }
+solana-version = { path = "../version", version = "=1.11.0" }
 systemstat = "0.1.10"
 tokio = { version = "1", features = ["full"] }
 
 [dev-dependencies]
-solana-logger = { path = "../logger", version = "=1.10.3" }
+solana-logger = { path = "../logger", version = "=1.11.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.10.3"
+version = "1.11.0"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -25,7 +25,9 @@ itertools = "0.10.2"
 jsonrpc-core = "18.0.0"
 lazy_static = "1.4.0"
 log = "0.4.14"
+lru = "0.7.5"
 quinn = "0.8.0"
+quinn-proto = "0.8.0"
 rand = "0.7.0"
 rand_chacha = "0.2.2"
 rayon = "1.5.1"
@@ -35,15 +37,17 @@ semver = "1.0.6"
 serde = "1.0.136"
 serde_derive = "1.0.103"
 serde_json = "1.0.79"
-solana-account-decoder = { path = "../account-decoder", version = "=1.10.3" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.10.3" }
-solana-faucet = { path = "../faucet", version = "=1.10.3" }
-solana-measure = { path = "../measure", version = "=1.10.3" }
-solana-net-utils = { path = "../net-utils", version = "=1.10.3" }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.3" }
-solana-version = { path = "../version", version = "=1.10.3" }
-solana-vote-program = { path = "../programs/vote", version = "=1.10.3" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.11.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.11.0" }
+solana-faucet = { path = "../faucet", version = "=1.11.0" }
+solana-measure = { path = "../measure", version = "=1.11.0" }
+solana-metrics = { path = "../metrics", version = "=1.11.0" }
+solana-net-utils = { path = "../net-utils", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
+solana-streamer = { path = "../streamer", version = "=1.11.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.11.0" }
+solana-version = { path = "../version", version = "=1.11.0" }
+solana-vote-program = { path = "../programs/vote", version = "=1.11.0" }
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 tokio-stream = "0.1.8"
@@ -52,9 +56,10 @@ tungstenite = { version = "0.17.2", features = ["rustls-tls-webpki-roots"] }
 url = "2.2.2"
 
 [dev-dependencies]
+anyhow = "1.0.45"
 assert_matches = "1.5.0"
 jsonrpc-http-server = "18.0.0"
-solana-logger = { path = "../logger", version = "=1.10.3" }
+solana-logger = { path = "../logger", version = "=1.11.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/connection_cache.rs

@@ -1,97 +1,329 @@
 use {
-    crate::{tpu_connection::TpuConnection, udp_client::UdpTpuConnection},
+    crate::{
+        quic_client::QuicTpuConnection,
+        tpu_connection::{ClientStats, TpuConnection},
+        udp_client::UdpTpuConnection,
+    },
     lazy_static::lazy_static,
+    lru::LruCache,
+    solana_net_utils::VALIDATOR_PORT_RANGE,
+    solana_sdk::{
+        timing::AtomicInterval, transaction::VersionedTransaction, transport::TransportError,
+    },
     std::{
-        collections::{hash_map::Entry, BTreeMap, HashMap},
-        net::{SocketAddr, UdpSocket},
-        sync::{Arc, Mutex},
+        net::{IpAddr, Ipv4Addr, SocketAddr},
+        sync::{
+            atomic::{AtomicU64, Ordering},
+            Arc, Mutex,
+        },
     },
 };
 
 // Should be non-zero
 static MAX_CONNECTIONS: usize = 64;
 
+#[derive(Clone)]
+enum Connection {
+    Udp(Arc<UdpTpuConnection>),
+    Quic(Arc<QuicTpuConnection>),
+}
+
+#[derive(Default)]
+struct ConnectionCacheStats {
+    cache_hits: AtomicU64,
+    cache_misses: AtomicU64,
+    sent_packets: AtomicU64,
+    total_batches: AtomicU64,
+    batch_success: AtomicU64,
+    batch_failure: AtomicU64,
+
+    // Need to track these separately per-connection
+    // because we need to track the base stat value from quinn
+    total_client_stats: ClientStats,
+}
+
+const CONNECTION_STAT_SUBMISSION_INTERVAL: u64 = 2000;
+
+impl ConnectionCacheStats {
+    fn add_client_stats(&self, client_stats: &ClientStats, num_packets: usize, is_success: bool) {
+        self.total_client_stats.total_connections.fetch_add(
+            client_stats.total_connections.load(Ordering::Relaxed),
+            Ordering::Relaxed,
+        );
+        self.total_client_stats.connection_reuse.fetch_add(
+            client_stats.connection_reuse.load(Ordering::Relaxed),
+            Ordering::Relaxed,
+        );
+        self.sent_packets
+            .fetch_add(num_packets as u64, Ordering::Relaxed);
+        self.total_batches.fetch_add(1, Ordering::Relaxed);
+        if is_success {
+            self.batch_success.fetch_add(1, Ordering::Relaxed);
+        } else {
+            self.batch_failure.fetch_add(1, Ordering::Relaxed);
+        }
+    }
+
+    fn report(&self) {
+        datapoint_info!(
+            "quic-client-connection-stats",
+            (
+                "cache_hits",
+                self.cache_hits.swap(0, Ordering::Relaxed),
+                i64
+            ),
+            (
+                "cache_misses",
+                self.cache_misses.swap(0, Ordering::Relaxed),
+                i64
+            ),
+            (
+                "total_connections",
+                self.total_client_stats
+                    .total_connections
+                    .swap(0, Ordering::Relaxed),
+                i64
+            ),
+            (
+                "connection_reuse",
+                self.total_client_stats
+                    .connection_reuse
+                    .swap(0, Ordering::Relaxed),
+                i64
+            ),
+            (
+                "congestion_events",
+                self.total_client_stats.congestion_events.load_and_reset(),
+                i64
+            ),
+            (
+                "tx_streams_blocked_uni",
+                self.total_client_stats
+                    .tx_streams_blocked_uni
+                    .load_and_reset(),
+                i64
+            ),
+            (
+                "tx_data_blocked",
+                self.total_client_stats.tx_data_blocked.load_and_reset(),
+                i64
+            ),
+            (
+                "tx_acks",
+                self.total_client_stats.tx_acks.load_and_reset(),
+                i64
+            ),
+        );
+    }
+}
+
 struct ConnMap {
-    // Keeps track of the connection associated with an addr and the last time it was used
-    map: HashMap<SocketAddr, (Arc<dyn TpuConnection + 'static + Sync + Send>, u64)>,
-    // Helps to find the least recently used connection. The search and inserts are O(log(n))
-    // but since we're bounding the size of the collections, this should be constant
-    // (and hopefully negligible) time. In theory, we can do this in constant time
-    // with a queue implemented as a doubly-linked list (and all the
-    // HashMap entries holding a "pointer" to the corresponding linked-list node),
-    // so we can push, pop and bump a used connection back to the end of the queue in O(1) time, but
-    // that seems non-"Rust-y" and low bang/buck. This is still pretty terrible though...
-    last_used_times: BTreeMap<u64, SocketAddr>,
-    ticks: u64,
+    map: LruCache<SocketAddr, Connection>,
+    stats: Arc<ConnectionCacheStats>,
+    last_stats: AtomicInterval,
+    use_quic: bool,
 }
 
 impl ConnMap {
     pub fn new() -> Self {
         Self {
-            map: HashMap::new(),
-            last_used_times: BTreeMap::new(),
-            ticks: 0,
+            map: LruCache::new(MAX_CONNECTIONS),
+            stats: Arc::new(ConnectionCacheStats::default()),
+            last_stats: AtomicInterval::default(),
+            use_quic: false,
         }
     }
+
+    pub fn set_use_quic(&mut self, use_quic: bool) {
+        self.use_quic = use_quic;
+    }
 }
 
 lazy_static! {
     static ref CONNECTION_MAP: Mutex<ConnMap> = Mutex::new(ConnMap::new());
 }
 
-#[allow(dead_code)]
+pub fn set_use_quic(use_quic: bool) {
+    let mut map = (*CONNECTION_MAP).lock().unwrap();
+    map.set_use_quic(use_quic);
+}
+
 // TODO: see https://github.com/solana-labs/solana/issues/23661
 // remove lazy_static and optimize and refactor this
-pub fn get_connection(addr: &SocketAddr) -> Arc<dyn TpuConnection + 'static + Sync + Send> {
+fn get_connection(addr: &SocketAddr) -> (Connection, Arc<ConnectionCacheStats>) {
     let mut map = (*CONNECTION_MAP).lock().unwrap();
-    let ticks = map.ticks;
 
-    let (conn, target_ticks) = match map.map.entry(*addr) {
-        Entry::Occupied(mut entry) => {
-            let mut pair = entry.get_mut();
-            let old_ticks = pair.1;
-            pair.1 = ticks;
-            (pair.0.clone(), old_ticks)
+    if map
+        .last_stats
+        .should_update(CONNECTION_STAT_SUBMISSION_INTERVAL)
+    {
+        map.stats.report();
     }
-        Entry::Vacant(entry) => {
-            let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
-            // TODO: see https://github.com/solana-labs/solana/issues/23659
-            // make it configurable (e.g. via the command line) whether to use UDP or Quic
-            let conn = Arc::new(UdpTpuConnection::new(send_socket, *addr));
-            entry.insert((conn.clone(), ticks));
-            (
-                conn as Arc<dyn TpuConnection + 'static + Sync + Send>,
-                ticks,
+
+    let (connection, hit, maybe_stats) = match map.map.get(addr) {
+        Some(connection) => {
+            let mut stats = None;
+            // update connection stats
+            if let Connection::Quic(conn) = connection {
+                stats = conn.stats().map(|s| (conn.base_stats(), s));
+            }
+            (connection.clone(), true, stats)
+        }
+        None => {
+            let (_, send_socket) = solana_net_utils::bind_in_range(
+                IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
+                VALIDATOR_PORT_RANGE,
             )
+            .unwrap();
+            let connection = if map.use_quic {
+                Connection::Quic(Arc::new(QuicTpuConnection::new(send_socket, *addr)))
+            } else {
+                Connection::Udp(Arc::new(UdpTpuConnection::new(send_socket, *addr)))
+            };
+
+            map.map.put(*addr, connection.clone());
+            (connection, false, None)
         }
     };
 
-    let num_connections = map.map.len();
-    if num_connections > MAX_CONNECTIONS {
-        let (old_ticks, target_addr) = {
-            let (old_ticks, target_addr) = map.last_used_times.iter().next().unwrap();
-            (*old_ticks, *target_addr)
+    if let Some((connection_stats, new_stats)) = maybe_stats {
+        map.stats.total_client_stats.congestion_events.update_stat(
+            &connection_stats.congestion_events,
+            new_stats.path.congestion_events,
+        );
+
+        map.stats
+            .total_client_stats
+            .tx_streams_blocked_uni
+            .update_stat(
+                &connection_stats.tx_streams_blocked_uni,
+                new_stats.frame_tx.streams_blocked_uni,
+            );
+
+        map.stats.total_client_stats.tx_data_blocked.update_stat(
+            &connection_stats.tx_data_blocked,
+            new_stats.frame_tx.data_blocked,
+        );
+
+        map.stats
+            .total_client_stats
+            .tx_acks
+            .update_stat(&connection_stats.tx_acks, new_stats.frame_tx.acks);
+    }
+
+    if hit {
+        map.stats.cache_hits.fetch_add(1, Ordering::Relaxed);
+    } else {
+        map.stats.cache_misses.fetch_add(1, Ordering::Relaxed);
+    }
+    (connection, map.stats.clone())
+}
+
+// TODO: see https://github.com/solana-labs/solana/issues/23851
+// use enum_dispatch and get rid of this tedious code.
+// The main blocker to using enum_dispatch right now is that
+// the it doesn't work with static methods like TpuConnection::new
+// which is used by thin_client. This will be eliminated soon
+// once thin_client is moved to using this connection cache.
+// Once that is done, we will migrate to using enum_dispatch
+// This will be done in a followup to
+// https://github.com/solana-labs/solana/pull/23817
+pub fn send_wire_transaction_batch(
+    packets: &[&[u8]],
+    addr: &SocketAddr,
+) -> Result<(), TransportError> {
+    let (conn, stats) = get_connection(addr);
+    let client_stats = ClientStats::default();
+    let r = match conn {
+        Connection::Udp(conn) => conn.send_wire_transaction_batch(packets, &client_stats),
+        Connection::Quic(conn) => conn.send_wire_transaction_batch(packets, &client_stats),
     };
-        map.map.remove(&target_addr);
-        map.last_used_times.remove(&old_ticks);
-    }
+    stats.add_client_stats(&client_stats, packets.len(), r.is_ok());
+    r
+}
 
-    if target_ticks != ticks {
-        map.last_used_times.remove(&target_ticks);
-    }
-    map.last_used_times.insert(ticks, *addr);
+pub fn send_wire_transaction_async(
+    packets: Vec<u8>,
+    addr: &SocketAddr,
+) -> Result<(), TransportError> {
+    let (conn, stats) = get_connection(addr);
+    let client_stats = Arc::new(ClientStats::default());
+    let r = match conn {
+        Connection::Udp(conn) => conn.send_wire_transaction_async(packets, client_stats.clone()),
+        Connection::Quic(conn) => conn.send_wire_transaction_async(packets, client_stats.clone()),
+    };
+    stats.add_client_stats(&client_stats, 1, r.is_ok());
+    r
+}
 
-    map.ticks += 1;
-    conn
+pub fn send_wire_transaction_batch_async(
+    packets: Vec<Vec<u8>>,
+    addr: &SocketAddr,
+) -> Result<(), TransportError> {
+    let (conn, stats) = get_connection(addr);
+    let client_stats = Arc::new(ClientStats::default());
+    let len = packets.len();
+    let r = match conn {
+        Connection::Udp(conn) => {
+            conn.send_wire_transaction_batch_async(packets, client_stats.clone())
+        }
+        Connection::Quic(conn) => {
+            conn.send_wire_transaction_batch_async(packets, client_stats.clone())
+        }
+    };
+    stats.add_client_stats(&client_stats, len, r.is_ok());
+    r
+}
+
+pub fn send_wire_transaction(
+    wire_transaction: &[u8],
+    addr: &SocketAddr,
+) -> Result<(), TransportError> {
+    send_wire_transaction_batch(&[wire_transaction], addr)
+}
+
+pub fn serialize_and_send_transaction(
+    transaction: &VersionedTransaction,
+    addr: &SocketAddr,
+) -> Result<(), TransportError> {
+    let (conn, stats) = get_connection(addr);
+    let client_stats = ClientStats::default();
+    let r = match conn {
+        Connection::Udp(conn) => conn.serialize_and_send_transaction(transaction, &client_stats),
+        Connection::Quic(conn) => conn.serialize_and_send_transaction(transaction, &client_stats),
+    };
+    stats.add_client_stats(&client_stats, 1, r.is_ok());
+    r
+}
+
+pub fn par_serialize_and_send_transaction_batch(
+    transactions: &[VersionedTransaction],
+    addr: &SocketAddr,
+) -> Result<(), TransportError> {
+    let (conn, stats) = get_connection(addr);
+    let client_stats = ClientStats::default();
+    let r = match conn {
+        Connection::Udp(conn) => {
+            conn.par_serialize_and_send_transaction_batch(transactions, &client_stats)
+        }
+        Connection::Quic(conn) => {
+            conn.par_serialize_and_send_transaction_batch(transactions, &client_stats)
+        }
+    };
+    stats.add_client_stats(&client_stats, transactions.len(), r.is_ok());
+    r
 }
 
 #[cfg(test)]
 mod tests {
     use {
-        crate::connection_cache::{get_connection, CONNECTION_MAP, MAX_CONNECTIONS},
+        crate::{
+            connection_cache::{get_connection, Connection, CONNECTION_MAP, MAX_CONNECTIONS},
+            tpu_connection::TpuConnection,
+        },
         rand::{Rng, SeedableRng},
         rand_chacha::ChaChaRng,
-        std::net::SocketAddr,
+        std::net::{IpAddr, SocketAddr},
     };
 
     fn get_addr(rng: &mut ChaChaRng) -> SocketAddr {
@@ -105,6 +337,13 @@ mod tests {
         addr_str.parse().expect("Invalid address")
     }
 
+    fn ip(conn: Connection) -> IpAddr {
+        match conn {
+            Connection::Udp(conn) => conn.tpu_addr().ip(),
+            Connection::Quic(conn) => conn.tpu_addr().ip(),
+        }
+    }
+
     #[test]
     fn test_connection_cache() {
         // Allow the test to run deterministically
@@ -120,7 +359,7 @@ mod tests {
         // be lazy and not connect until first use or handle connection errors somehow
         // (without crashing, as would be required in a real practical validator)
         let first_addr = get_addr(&mut rng);
-        assert!(get_connection(&first_addr).tpu_addr().ip() == first_addr.ip());
+        assert!(ip(get_connection(&first_addr).0) == first_addr.ip());
         let addrs = (0..MAX_CONNECTIONS)
             .into_iter()
             .map(|_| {
@@ -132,11 +371,11 @@ mod tests {
         {
             let map = (*CONNECTION_MAP).lock().unwrap();
             addrs.iter().for_each(|a| {
-                let conn = map.map.get(a).expect("Address not found");
-                assert!(a.ip() == conn.0.tpu_addr().ip());
+                let conn = map.map.peek(a).expect("Address not found");
+                assert!(a.ip() == ip(conn.clone()));
             });
 
-            assert!(map.map.get(&first_addr).is_none());
+            assert!(map.map.peek(&first_addr).is_none());
         }
 
         // Test that get_connection updates which connection is next up for eviction
@@ -149,7 +388,7 @@ mod tests {
         get_connection(&get_addr(&mut rng));
 
         let map = (*CONNECTION_MAP).lock().unwrap();
-        assert!(map.map.get(&addrs[0]).is_some());
-        assert!(map.map.get(&addrs[1]).is_none());
+        assert!(map.map.peek(&addrs[0]).is_some());
+        assert!(map.map.peek(&addrs[1]).is_none());
     }
 }

client/src/http_sender.rs

@@ -197,6 +197,10 @@ impl RpcSender for HttpSender {
             return Ok(json["result"].take());
         }
     }
+
+    fn url(&self) -> String {
+        self.url.clone()
+    }
 }
 
 #[cfg(test)]

client/src/lib.rs

@@ -2,6 +2,9 @@
 #[macro_use]
 extern crate serde_derive;
 
+#[macro_use]
+extern crate solana_metrics;
+
 pub mod blockhash_query;
 pub mod client_error;
 pub mod connection_cache;
@@ -9,7 +12,6 @@ pub(crate) mod http_sender;
 pub(crate) mod mock_sender;
 pub mod nonblocking;
 pub mod nonce_utils;
-pub mod perf_utils;
 pub mod pubsub_client;
 pub mod quic_client;
 pub mod rpc_cache;

client/src/mock_sender.rs

@@ -229,6 +229,7 @@ impl RpcSender for MockSender {
                             post_token_balances: None,
                             rewards: None,
                             loaded_addresses: None,
+                            return_data: None,
                         }),
                 },
                 block_time: Some(1628633791),
@@ -340,6 +341,7 @@ impl RpcSender for MockSender {
                     logs: None,
                     accounts: None,
                     units_consumed: None,
+                    return_data: None,
                 },
             })?,
             "getMinimumBalanceForRentExemption" => json![20],
@@ -468,4 +470,8 @@ impl RpcSender for MockSender {
         };
         Ok(val)
     }
+
+    fn url(&self) -> String {
+        format!("MockSender: {}", self.url)
+    }
 }

client/src/nonblocking/rpc_client.rs

@@ -502,6 +502,11 @@ impl RpcClient {
         Self::new_with_timeout(url, timeout)
     }
 
+    /// Get the configured url of the client's sender
+    pub fn url(&self) -> String {
+        self.sender.url()
+    }
+
     async fn get_node_version(&self) -> Result<semver::Version, RpcError> {
         let r_node_version = self.node_version.read().await;
         if let Some(version) = &*r_node_version {
@@ -3727,6 +3732,61 @@ impl RpcClient {
             commitment: Some(self.maybe_map_commitment(commitment_config).await?),
             data_slice: None,
         };
+
+        self.get_account_with_config(pubkey, config).await
+    }
+
+    /// Returns all information associated with the account of the provided pubkey.
+    ///
+    /// If the account does not exist, this method returns `Ok(None)`.
+    ///
+    /// To get multiple accounts at once, use the [`get_multiple_accounts_with_config`] method.
+    ///
+    /// [`get_multiple_accounts_with_config`]: RpcClient::get_multiple_accounts_with_config
+    ///
+    /// # RPC Reference
+    ///
+    /// This method is built on the [`getAccountInfo`] RPC method.
+    ///
+    /// [`getAccountInfo`]: https://docs.solana.com/developing/clients/jsonrpc-api#getaccountinfo
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use solana_client::{
+    /// #     rpc_client::{self, RpcClient},
+    /// #     rpc_config::RpcAccountInfoConfig,
+    /// #     client_error::ClientError,
+    /// # };
+    /// # use solana_sdk::{
+    /// #     signature::Signer,
+    /// #     signer::keypair::Keypair,
+    /// #     pubkey::Pubkey,
+    /// #     commitment_config::CommitmentConfig,
+    /// # };
+    /// # use solana_account_decoder::UiAccountEncoding;
+    /// # use std::str::FromStr;
+    /// # let mocks = rpc_client::create_rpc_client_mocks();
+    /// # let rpc_client = RpcClient::new_mock_with_mocks("succeeds".to_string(), mocks);
+    /// let alice_pubkey = Pubkey::from_str("BgvYtJEfmZYdVKiptmMjxGzv8iQoo4MWjsP3QsTkhhxa").unwrap();
+    /// let commitment_config = CommitmentConfig::processed();
+    /// let config = RpcAccountInfoConfig {
+    ///     encoding: Some(UiAccountEncoding::Base64),
+    ///     commitment: Some(commitment_config),
+    ///     .. RpcAccountInfoConfig::default()
+    /// };
+    /// let account = rpc_client.get_account_with_config(
+    ///     &alice_pubkey,
+    ///     config,
+    /// )?;
+    /// assert!(account.value.is_some());
+    /// # Ok::<(), ClientError>(())
+    /// ```
+    pub async fn get_account_with_config(
+        &self,
+        pubkey: &Pubkey,
+        config: RpcAccountInfoConfig,
+    ) -> RpcResult<Option<Account>> {
         let response = self
             .send(
                 RpcRequest::GetAccountInfo,

client/src/nonce_utils.rs

@@ -1,3 +1,5 @@
+//! Durable transaction nonce helpers.
+
 use {
     crate::rpc_client::RpcClient,
     solana_sdk::{
@@ -32,10 +34,23 @@ pub enum Error {
     Client(String),
 }
 
+/// Get a nonce account from the network.
+///
+/// This is like [`RpcClient::get_account`] except:
+///
+/// - it returns this module's [`Error`] type,
+/// - it returns an error if any of the checks from [`account_identity_ok`] fail.
 pub fn get_account(rpc_client: &RpcClient, nonce_pubkey: &Pubkey) -> Result<Account, Error> {
     get_account_with_commitment(rpc_client, nonce_pubkey, CommitmentConfig::default())
 }
 
+/// Get a nonce account from the network.
+///
+/// This is like [`RpcClient::get_account_with_commitment`] except:
+///
+/// - it returns this module's [`Error`] type,
+/// - it returns an error if the account does not exist,
+/// - it returns an error if any of the checks from [`account_identity_ok`] fail.
 pub fn get_account_with_commitment(
     rpc_client: &RpcClient,
     nonce_pubkey: &Pubkey,
@@ -52,6 +67,13 @@ pub fn get_account_with_commitment(
         .and_then(|a| account_identity_ok(&a).map(|()| a))
 }
 
+/// Perform basic checks that an account has nonce-like properties.
+///
+/// # Errors
+///
+/// Returns [`Error::InvalidAccountOwner`] if the account is not owned by the
+/// system program. Returns [`Error::UnexpectedDataSize`] if the account
+/// contains no data.
 pub fn account_identity_ok<T: ReadableAccount>(account: &T) -> Result<(), Error> {
     if account.owner() != &system_program::id() {
         Err(Error::InvalidAccountOwner)
@@ -62,6 +84,47 @@ pub fn account_identity_ok<T: ReadableAccount>(account: &T) -> Result<(), Error>
     }
 }
 
+/// Deserialize the state of a durable transaction nonce account.
+///
+/// # Errors
+///
+/// Returns an error if the account is not owned by the system program or
+/// contains no data.
+///
+/// # Examples
+///
+/// Determine if a nonce account is initialized:
+///
+/// ```no_run
+/// use solana_client::{
+///     rpc_client::RpcClient,
+///     nonce_utils,
+/// };
+/// use solana_sdk::{
+///     nonce::State,
+///     pubkey::Pubkey,
+/// };
+/// use anyhow::Result;
+///
+/// fn is_nonce_initialized(
+///     client: &RpcClient,
+///     nonce_account_pubkey: &Pubkey,
+/// ) -> Result<bool> {
+///
+///     // Sign the tx with nonce_account's `blockhash` instead of the
+///     // network's latest blockhash.
+///     let nonce_account = client.get_account(nonce_account_pubkey)?;
+///     let nonce_state = nonce_utils::state_from_account(&nonce_account)?;
+///
+///     Ok(!matches!(nonce_state, State::Uninitialized))
+/// }
+/// #
+/// # let client = RpcClient::new(String::new());
+/// # let nonce_account_pubkey = Pubkey::new_unique();
+/// # is_nonce_initialized(&client, &nonce_account_pubkey)?;
+/// #
+/// # Ok::<(), anyhow::Error>(())
+/// ```
 pub fn state_from_account<T: ReadableAccount + StateMut<Versions>>(
     account: &T,
 ) -> Result<State, Error> {
@@ -71,6 +134,93 @@ pub fn state_from_account<T: ReadableAccount + StateMut<Versions>>(
         .map(|v| v.convert_to_current())
 }
 
+/// Deserialize the state data of a durable transaction nonce account.
+///
+/// # Errors
+///
+/// Returns an error if the account is not owned by the system program or
+/// contains no data. Returns an error if the account state is uninitialized or
+/// fails to deserialize.
+///
+/// # Examples
+///
+/// Create and sign a transaction with a durable nonce:
+///
+/// ```no_run
+/// use solana_client::{
+///     rpc_client::RpcClient,
+///     nonce_utils,
+/// };
+/// use solana_sdk::{
+///     message::Message,
+///     pubkey::Pubkey,
+///     signature::{Keypair, Signer},
+///     system_instruction,
+///     transaction::Transaction,
+/// };
+/// use std::path::Path;
+/// use anyhow::Result;
+/// # use anyhow::anyhow;
+///
+/// fn create_transfer_tx_with_nonce(
+///     client: &RpcClient,
+///     nonce_account_pubkey: &Pubkey,
+///     payer: &Keypair,
+///     receiver: &Pubkey,
+///     amount: u64,
+///     tx_path: &Path,
+/// ) -> Result<()> {
+///
+///     let instr_transfer = system_instruction::transfer(
+///         &payer.pubkey(),
+///         receiver,
+///         amount,
+///     );
+///
+///     // In this example, `payer` is `nonce_account_pubkey`'s authority
+///     let instr_advance_nonce_account = system_instruction::advance_nonce_account(
+///         nonce_account_pubkey,
+///         &payer.pubkey(),
+///     );
+///
+///     // The `advance_nonce_account` instruction must be the first issued in
+///     // the transaction.
+///     let message = Message::new(
+///         &[
+///             instr_advance_nonce_account,
+///             instr_transfer
+///         ],
+///         Some(&payer.pubkey()),
+///     );
+///
+///     let mut tx = Transaction::new_unsigned(message);
+///
+///     // Sign the tx with nonce_account's `blockhash` instead of the
+///     // network's latest blockhash.
+///     let nonce_account = client.get_account(nonce_account_pubkey)?;
+///     let nonce_data = nonce_utils::data_from_account(&nonce_account)?;
+///     let blockhash = nonce_data.blockhash;
+///
+///     tx.try_sign(&[payer], blockhash)?;
+///
+///     // Save the signed transaction locally for later submission.
+///     save_tx_to_file(&tx_path, &tx)?;
+///
+///     Ok(())
+/// }
+/// #
+/// # fn save_tx_to_file(path: &Path, tx: &Transaction) -> Result<()> {
+/// #     Ok(())
+/// # }
+/// #
+/// # let client = RpcClient::new(String::new());
+/// # let nonce_account_pubkey = Pubkey::new_unique();
+/// # let payer = Keypair::new();
+/// # let receiver = Pubkey::new_unique();
+/// # create_transfer_tx_with_nonce(&client, &nonce_account_pubkey, &payer, &receiver, 1024, Path::new("new_tx"))?;
+/// #
+/// # Ok::<(), anyhow::Error>(())
+/// ```
 pub fn data_from_account<T: ReadableAccount + StateMut<Versions>>(
     account: &T,
 ) -> Result<Data, Error> {
@@ -78,6 +228,12 @@ pub fn data_from_account<T: ReadableAccount + StateMut<Versions>>(
     state_from_account(account).and_then(|ref s| data_from_state(s).map(|d| d.clone()))
 }
 
+/// Get the nonce data from its [`State`] value.
+///
+/// # Errors
+///
+/// Returns [`Error::InvalidStateForOperation`] if `state` is
+/// [`State::Uninitialized`].
 pub fn data_from_state(state: &State) -> Result<&Data, Error> {
     match state {
         State::Uninitialized => Err(Error::InvalidStateForOperation),

client/src/quic_client.rs

@@ -2,20 +2,24 @@
 //! an interface for sending transactions which is restricted by the server's flow control.
 
 use {
-    crate::{client_error::ClientErrorKind, tpu_connection::TpuConnection},
+    crate::{
+        client_error::ClientErrorKind,
+        tpu_connection::{ClientStats, TpuConnection},
+    },
     async_mutex::Mutex,
     futures::future::join_all,
     itertools::Itertools,
+    lazy_static::lazy_static,
+    log::*,
     quinn::{ClientConfig, Endpoint, EndpointConfig, NewConnection, WriteError},
-    rayon::iter::{IntoParallelIterator, ParallelIterator},
+    quinn_proto::ConnectionStats,
     solana_sdk::{
         quic::{QUIC_MAX_CONCURRENT_STREAMS, QUIC_PORT_OFFSET},
-        transaction::Transaction,
         transport::Result as TransportResult,
     },
     std::{
         net::{SocketAddr, UdpSocket},
-        sync::Arc,
+        sync::{atomic::Ordering, Arc},
     },
     tokio::runtime::Runtime,
 };
@@ -41,18 +45,34 @@ impl rustls::client::ServerCertVerifier for SkipServerVerification {
         Ok(rustls::client::ServerCertVerified::assertion())
     }
 }
+lazy_static! {
+    static ref RUNTIME: Runtime = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+}
 
 struct QuicClient {
-    runtime: Runtime,
     endpoint: Endpoint,
     connection: Arc<Mutex<Option<Arc<NewConnection>>>>,
     addr: SocketAddr,
+    stats: Arc<ClientStats>,
 }
 
 pub struct QuicTpuConnection {
     client: Arc<QuicClient>,
 }
 
+impl QuicTpuConnection {
+    pub fn stats(&self) -> Option<ConnectionStats> {
+        self.client.stats()
+    }
+
+    pub fn base_stats(&self) -> Arc<ClientStats> {
+        self.client.stats.clone()
+    }
+}
+
 impl TpuConnection for QuicTpuConnection {
     fn new(client_socket: UdpSocket, tpu_addr: SocketAddr) -> Self {
         let tpu_addr = SocketAddr::new(tpu_addr.ip(), tpu_addr.port() + QUIC_PORT_OFFSET);
@@ -65,34 +85,74 @@ impl TpuConnection for QuicTpuConnection {
         &self.client.addr
     }
 
-    fn send_wire_transaction(&self, data: Vec<u8>) -> TransportResult<()> {
-        let _guard = self.client.runtime.enter();
-        let send_buffer = self.client.send_buffer(&data[..]);
-        self.client.runtime.block_on(send_buffer)?;
+    fn send_wire_transaction<T>(
+        &self,
+        wire_transaction: T,
+        stats: &ClientStats,
+    ) -> TransportResult<()>
+    where
+        T: AsRef<[u8]>,
+    {
+        let _guard = RUNTIME.enter();
+        let send_buffer = self.client.send_buffer(wire_transaction, stats);
+        RUNTIME.block_on(send_buffer)?;
         Ok(())
     }
 
-    fn send_batch(&self, transactions: Vec<Transaction>) -> TransportResult<()> {
-        let buffers = transactions
-            .into_par_iter()
-            .map(|tx| bincode::serialize(&tx).expect("serialize Transaction in send_batch"))
-            .collect::<Vec<_>>();
+    fn send_wire_transaction_batch<T>(
+        &self,
+        buffers: &[T],
+        stats: &ClientStats,
+    ) -> TransportResult<()>
+    where
+        T: AsRef<[u8]>,
+    {
+        let _guard = RUNTIME.enter();
+        let send_batch = self.client.send_batch(buffers, stats);
+        RUNTIME.block_on(send_batch)?;
+        Ok(())
+    }
 
-        let _guard = self.client.runtime.enter();
-        let send_batch = self.client.send_batch(&buffers[..]);
-        self.client.runtime.block_on(send_batch)?;
+    fn send_wire_transaction_async(
+        &self,
+        wire_transaction: Vec<u8>,
+        stats: Arc<ClientStats>,
+    ) -> TransportResult<()> {
+        let _guard = RUNTIME.enter();
+        let client = self.client.clone();
+        //drop and detach the task
+        let _ = RUNTIME.spawn(async move {
+            let send_buffer = client.send_buffer(wire_transaction, &stats);
+            if let Err(e) = send_buffer.await {
+                warn!("Failed to send transaction async to {:?}", e);
+                datapoint_warn!("send-wire-async", ("failure", 1, i64),);
+            }
+        });
+        Ok(())
+    }
+
+    fn send_wire_transaction_batch_async(
+        &self,
+        buffers: Vec<Vec<u8>>,
+        stats: Arc<ClientStats>,
+    ) -> TransportResult<()> {
+        let _guard = RUNTIME.enter();
+        let client = self.client.clone();
+        //drop and detach the task
+        let _ = RUNTIME.spawn(async move {
+            let send_batch = client.send_batch(&buffers, &stats);
+            if let Err(e) = send_batch.await {
+                warn!("Failed to send transaction batch async to {:?}", e);
+                datapoint_warn!("send-wire-batch-async", ("failure", 1, i64),);
+            }
+        });
         Ok(())
     }
 }
 
 impl QuicClient {
     pub fn new(client_socket: UdpSocket, addr: SocketAddr) -> Self {
-        let runtime = tokio::runtime::Builder::new_multi_thread()
-            .enable_all()
-            .build()
-            .unwrap();
-
-        let _guard = runtime.enter();
+        let _guard = RUNTIME.enter();
 
         let crypto = rustls::ClientConfig::builder()
             .with_safe_defaults()
@@ -101,18 +161,24 @@ impl QuicClient {
 
         let create_endpoint = QuicClient::create_endpoint(EndpointConfig::default(), client_socket);
 
-        let mut endpoint = runtime.block_on(create_endpoint);
+        let mut endpoint = RUNTIME.block_on(create_endpoint);
 
         endpoint.set_default_client_config(ClientConfig::new(Arc::new(crypto)));
 
         Self {
-            runtime,
             endpoint,
             connection: Arc::new(Mutex::new(None)),
             addr,
+            stats: Arc::new(ClientStats::default()),
         }
     }
 
+    pub fn stats(&self) -> Option<ConnectionStats> {
+        let conn_guard = self.connection.lock();
+        let x = RUNTIME.block_on(conn_guard);
+        x.as_ref().map(|c| c.connection.stats())
+    }
+
     // If this function becomes public, it should be changed to
     // not expose details of the specific Quic implementation we're using
     async fn create_endpoint(config: EndpointConfig, client_socket: UdpSocket) -> Endpoint {
@@ -129,18 +195,35 @@ impl QuicClient {
         Ok(())
     }
 
+    async fn make_connection(&self, stats: &ClientStats) -> Result<Arc<NewConnection>, WriteError> {
+        let connecting = self.endpoint.connect(self.addr, "connect").unwrap();
+        stats.total_connections.fetch_add(1, Ordering::Relaxed);
+        let connecting_result = connecting.await;
+        if connecting_result.is_err() {
+            stats.connection_errors.fetch_add(1, Ordering::Relaxed);
+        }
+        let connection = connecting_result?;
+        Ok(Arc::new(connection))
+    }
+
     // Attempts to send data, connecting/reconnecting as necessary
     // On success, returns the connection used to successfully send the data
-    async fn _send_buffer(&self, data: &[u8]) -> Result<Arc<NewConnection>, WriteError> {
+    async fn _send_buffer(
+        &self,
+        data: &[u8],
+        stats: &ClientStats,
+    ) -> Result<Arc<NewConnection>, WriteError> {
         let connection = {
             let mut conn_guard = self.connection.lock().await;
 
             let maybe_conn = (*conn_guard).clone();
             match maybe_conn {
-                Some(conn) => conn.clone(),
+                Some(conn) => {
+                    stats.connection_reuse.fetch_add(1, Ordering::Relaxed);
+                    conn.clone()
+                }
                 None => {
-                    let connecting = self.endpoint.connect(self.addr, "connect").unwrap();
-                    let connection = Arc::new(connecting.await?);
+                    let connection = self.make_connection(stats).await?;
                     *conn_guard = Some(connection.clone());
                     connection
                 }
@@ -150,8 +233,7 @@ impl QuicClient {
             Ok(()) => Ok(connection),
             _ => {
                 let connection = {
-                    let connecting = self.endpoint.connect(self.addr, "connect").unwrap();
-                    let connection = Arc::new(connecting.await?);
+                    let connection = self.make_connection(stats).await?;
                     let mut conn_guard = self.connection.lock().await;
                     *conn_guard = Some(connection.clone());
                     connection
@@ -162,12 +244,22 @@ impl QuicClient {
         }
     }
 
-    pub async fn send_buffer(&self, data: &[u8]) -> Result<(), ClientErrorKind> {
-        self._send_buffer(data).await?;
+    pub async fn send_buffer<T>(&self, data: T, stats: &ClientStats) -> Result<(), ClientErrorKind>
+    where
+        T: AsRef<[u8]>,
+    {
+        self._send_buffer(data.as_ref(), stats).await?;
         Ok(())
     }
 
-    pub async fn send_batch(&self, buffers: &[Vec<u8>]) -> Result<(), ClientErrorKind> {
+    pub async fn send_batch<T>(
+        &self,
+        buffers: &[T],
+        stats: &ClientStats,
+    ) -> Result<(), ClientErrorKind>
+    where
+        T: AsRef<[u8]>,
+    {
         // Start off by "testing" the connection by sending the first transaction
         // This will also connect to the server if not already connected
         // and reconnect and retry if the first send attempt failed
@@ -182,7 +274,7 @@ impl QuicClient {
         if buffers.is_empty() {
             return Ok(());
         }
-        let connection = self._send_buffer(&buffers[0][..]).await?;
+        let connection = self._send_buffer(buffers[0].as_ref(), stats).await?;
 
         // Used to avoid dereferencing the Arc multiple times below
         // by just getting a reference to the NewConnection once
@@ -192,13 +284,16 @@ impl QuicClient {
             .iter()
             .chunks(QUIC_MAX_CONCURRENT_STREAMS);
 
-        let futures = chunks.into_iter().map(|buffs| {
+        let futures: Vec<_> = chunks
+            .into_iter()
+            .map(|buffs| {
                 join_all(
                     buffs
                         .into_iter()
-                    .map(|buf| Self::_send_buffer_using_conn(&buf[..], connection_ref)),
+                        .map(|buf| Self::_send_buffer_using_conn(buf.as_ref(), connection_ref)),
                 )
-        });
+            })
+            .collect();
 
         for f in futures {
             f.await.into_iter().try_for_each(|res| res)?;

client/src/rpc_client.rs

@@ -535,6 +535,11 @@ impl RpcClient {
         Self::new_with_timeout(url, timeout)
     }
 
+    /// Get the configured url of the client's sender
+    pub fn url(&self) -> String {
+        self.rpc_client.url()
+    }
+
     /// Get the configured default [commitment level][cl].
     ///
     /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
@@ -3245,6 +3250,60 @@ impl RpcClient {
         )
     }
 
+    /// Returns all information associated with the account of the provided pubkey.
+    ///
+    /// If the account does not exist, this method returns `Ok(None)`.
+    ///
+    /// To get multiple accounts at once, use the [`get_multiple_accounts_with_config`] method.
+    ///
+    /// [`get_multiple_accounts_with_config`]: RpcClient::get_multiple_accounts_with_config
+    ///
+    /// # RPC Reference
+    ///
+    /// This method is built on the [`getAccountInfo`] RPC method.
+    ///
+    /// [`getAccountInfo`]: https://docs.solana.com/developing/clients/jsonrpc-api#getaccountinfo
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use solana_client::{
+    /// #     rpc_client::{self, RpcClient},
+    /// #     rpc_config::RpcAccountInfoConfig,
+    /// #     client_error::ClientError,
+    /// # };
+    /// # use solana_sdk::{
+    /// #     signature::Signer,
+    /// #     signer::keypair::Keypair,
+    /// #     pubkey::Pubkey,
+    /// #     commitment_config::CommitmentConfig,
+    /// # };
+    /// # use solana_account_decoder::UiAccountEncoding;
+    /// # use std::str::FromStr;
+    /// # let mocks = rpc_client::create_rpc_client_mocks();
+    /// # let rpc_client = RpcClient::new_mock_with_mocks("succeeds".to_string(), mocks);
+    /// let alice_pubkey = Pubkey::from_str("BgvYtJEfmZYdVKiptmMjxGzv8iQoo4MWjsP3QsTkhhxa").unwrap();
+    /// let commitment_config = CommitmentConfig::processed();
+    /// let config = RpcAccountInfoConfig {
+    ///     encoding: Some(UiAccountEncoding::Base64),
+    ///     commitment: Some(commitment_config),
+    ///     .. RpcAccountInfoConfig::default()
+    /// };
+    /// let account = rpc_client.get_account_with_config(
+    ///     &alice_pubkey,
+    ///     config,
+    /// )?;
+    /// assert!(account.value.is_some());
+    /// # Ok::<(), ClientError>(())
+    /// ```
+    pub fn get_account_with_config(
+        &self,
+        pubkey: &Pubkey,
+        config: RpcAccountInfoConfig,
+    ) -> RpcResult<Option<Account>> {
+        self.invoke(self.rpc_client.get_account_with_config(pubkey, config))
+    }
+
     /// Get the max slot seen from retransmit stage.
     ///
     /// # RPC Reference

client/src/rpc_response.rs

@@ -7,6 +7,7 @@ use {
         hash::Hash,
         inflation::Inflation,
         transaction::{Result, TransactionError},
+        transaction_context::TransactionReturnData,
     },
     solana_transaction_status::{
         ConfirmedTransactionStatusWithSignature, TransactionConfirmationStatus, UiConfirmedBlock,
@@ -347,6 +348,29 @@ pub struct RpcSimulateTransactionResult {
     pub logs: Option<Vec<String>>,
     pub accounts: Option<Vec<Option<UiAccount>>>,
     pub units_consumed: Option<u64>,
+    pub return_data: Option<RpcTransactionReturnData>,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcTransactionReturnData {
+    pub program_id: String,
+    pub data: (String, ReturnDataEncoding),
+}
+
+impl From<TransactionReturnData> for RpcTransactionReturnData {
+    fn from(return_data: TransactionReturnData) -> Self {
+        Self {
+            program_id: return_data.program_id.to_string(),
+            data: (base64::encode(return_data.data), ReturnDataEncoding::Base64),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Copy, Debug, Eq, Hash, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub enum ReturnDataEncoding {
+    Base64,
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug)]

client/src/rpc_sender.rs

@@ -32,4 +32,5 @@ pub trait RpcSender {
         params: serde_json::Value,
     ) -> Result<serde_json::Value>;
     fn get_transport_stats(&self) -> RpcTransportStats;
+    fn url(&self) -> String;
 }

client/src/thin_client.rs

@@ -5,8 +5,13 @@
 
 use {
     crate::{
-        rpc_client::RpcClient, rpc_config::RpcProgramAccountsConfig, rpc_response::Response,
-        tpu_connection::TpuConnection, udp_client::UdpTpuConnection,
+        connection_cache::{
+            par_serialize_and_send_transaction_batch, send_wire_transaction,
+            serialize_and_send_transaction,
+        },
+        rpc_client::RpcClient,
+        rpc_config::RpcProgramAccountsConfig,
+        rpc_response::Response,
     },
     log::*,
     solana_sdk::{
@@ -24,12 +29,12 @@ use {
         signers::Signers,
         system_instruction,
         timing::duration_as_ms,
-        transaction::{self, Transaction},
+        transaction::{self, Transaction, VersionedTransaction},
         transport::Result as TransportResult,
     },
     std::{
         io,
-        net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
+        net::SocketAddr,
         sync::{
             atomic::{AtomicBool, AtomicUsize, Ordering},
             RwLock,
@@ -118,67 +123,55 @@ impl ClientOptimizer {
 }
 
 /// An object for querying and sending transactions to the network.
-pub struct ThinClient<C: 'static + TpuConnection> {
+pub struct ThinClient {
     rpc_clients: Vec<RpcClient>,
-    tpu_connections: Vec<C>,
+    tpu_addrs: Vec<SocketAddr>,
     optimizer: ClientOptimizer,
 }
 
-impl<C: 'static + TpuConnection> ThinClient<C> {
+impl ThinClient {
     /// Create a new ThinClient that will interface with the Rpc at `rpc_addr` using TCP
     /// and the Tpu at `tpu_addr` over `transactions_socket` using Quic or UDP
     /// (currently hardcoded to UDP)
-    pub fn new(rpc_addr: SocketAddr, tpu_addr: SocketAddr, transactions_socket: UdpSocket) -> Self {
-        let tpu_connection = C::new(transactions_socket, tpu_addr);
-
-        Self::new_from_client(RpcClient::new_socket(rpc_addr), tpu_connection)
+    pub fn new(rpc_addr: SocketAddr, tpu_addr: SocketAddr) -> Self {
+        Self::new_from_client(RpcClient::new_socket(rpc_addr), tpu_addr)
     }
 
     pub fn new_socket_with_timeout(
         rpc_addr: SocketAddr,
         tpu_addr: SocketAddr,
-        transactions_socket: UdpSocket,
         timeout: Duration,
     ) -> Self {
         let rpc_client = RpcClient::new_socket_with_timeout(rpc_addr, timeout);
-        let tpu_connection = C::new(transactions_socket, tpu_addr);
-        Self::new_from_client(rpc_client, tpu_connection)
+        Self::new_from_client(rpc_client, tpu_addr)
     }
 
-    fn new_from_client(rpc_client: RpcClient, tpu_connection: C) -> Self {
+    fn new_from_client(rpc_client: RpcClient, tpu_addr: SocketAddr) -> Self {
         Self {
             rpc_clients: vec![rpc_client],
-            tpu_connections: vec![tpu_connection],
+            tpu_addrs: vec![tpu_addr],
             optimizer: ClientOptimizer::new(0),
         }
     }
 
-    pub fn new_from_addrs(
-        rpc_addrs: Vec<SocketAddr>,
-        tpu_addrs: Vec<SocketAddr>,
-        transactions_socket: UdpSocket,
-    ) -> Self {
+    pub fn new_from_addrs(rpc_addrs: Vec<SocketAddr>, tpu_addrs: Vec<SocketAddr>) -> Self {
         assert!(!rpc_addrs.is_empty());
         assert_eq!(rpc_addrs.len(), tpu_addrs.len());
 
         let rpc_clients: Vec<_> = rpc_addrs.into_iter().map(RpcClient::new_socket).collect();
         let optimizer = ClientOptimizer::new(rpc_clients.len());
-        let tpu_connections: Vec<_> = tpu_addrs
-            .into_iter()
-            .map(|tpu_addr| C::new(transactions_socket.try_clone().unwrap(), tpu_addr))
-            .collect();
         Self {
             rpc_clients,
-            tpu_connections,
+            tpu_addrs,
             optimizer,
         }
     }
 
-    fn tpu_connection(&self) -> &C {
-        &self.tpu_connections[self.optimizer.best()]
+    fn tpu_addr(&self) -> &SocketAddr {
+        &self.tpu_addrs[self.optimizer.best()]
     }
 
-    fn rpc_client(&self) -> &RpcClient {
+    pub fn rpc_client(&self) -> &RpcClient {
         &self.rpc_clients[self.optimizer.best()]
     }
 
@@ -215,10 +208,12 @@ impl<C: 'static + TpuConnection> ThinClient<C> {
             let mut num_confirmed = 0;
             let mut wait_time = MAX_PROCESSING_AGE;
             // resend the same transaction until the transaction has no chance of succeeding
+            let wire_transaction =
+                bincode::serialize(&transaction).expect("transaction serialization failed");
             while now.elapsed().as_secs() < wait_time as u64 {
                 if num_confirmed == 0 {
                     // Send the transaction if there has been no confirmation (e.g. the first time)
-                    self.tpu_connection().send_transaction(transaction)?;
+                    send_wire_transaction(&wire_transaction, self.tpu_addr())?;
                 }
 
                 if let Ok(confirmed_blocks) = self.poll_for_signature_confirmation(
@@ -313,13 +308,13 @@ impl<C: 'static + TpuConnection> ThinClient<C> {
     }
 }
 
-impl<C: 'static + TpuConnection> Client for ThinClient<C> {
+impl Client for ThinClient {
     fn tpu_addr(&self) -> String {
-        self.tpu_connection().tpu_addr().to_string()
+        self.tpu_addr().to_string()
     }
 }
 
-impl<C: 'static + TpuConnection> SyncClient for ThinClient<C> {
+impl SyncClient for ThinClient {
     fn send_and_confirm_message<T: Signers>(
         &self,
         keypairs: &T,
@@ -599,64 +594,34 @@ impl<C: 'static + TpuConnection> SyncClient for ThinClient<C> {
     }
 }
 
-impl<C: 'static + TpuConnection> AsyncClient for ThinClient<C> {
-    fn async_send_transaction(&self, transaction: Transaction) -> TransportResult<Signature> {
-        self.tpu_connection().send_transaction(&transaction)?;
+impl AsyncClient for ThinClient {
+    fn async_send_versioned_transaction(
+        &self,
+        transaction: VersionedTransaction,
+    ) -> TransportResult<Signature> {
+        serialize_and_send_transaction(&transaction, self.tpu_addr())?;
         Ok(transaction.signatures[0])
     }
 
-    fn async_send_batch(&self, transactions: Vec<Transaction>) -> TransportResult<()> {
-        self.tpu_connection().send_batch(transactions)
-    }
-
-    fn async_send_message<T: Signers>(
+    fn async_send_versioned_transaction_batch(
         &self,
-        keypairs: &T,
-        message: Message,
-        recent_blockhash: Hash,
-    ) -> TransportResult<Signature> {
-        let transaction = Transaction::new(keypairs, message, recent_blockhash);
-        self.async_send_transaction(transaction)
-    }
-    fn async_send_instruction(
-        &self,
-        keypair: &Keypair,
-        instruction: Instruction,
-        recent_blockhash: Hash,
-    ) -> TransportResult<Signature> {
-        let message = Message::new(&[instruction], Some(&keypair.pubkey()));
-        self.async_send_message(&[keypair], message, recent_blockhash)
-    }
-    fn async_transfer(
-        &self,
-        lamports: u64,
-        keypair: &Keypair,
-        pubkey: &Pubkey,
-        recent_blockhash: Hash,
-    ) -> TransportResult<Signature> {
-        let transfer_instruction =
-            system_instruction::transfer(&keypair.pubkey(), pubkey, lamports);
-        self.async_send_instruction(keypair, transfer_instruction, recent_blockhash)
+        batch: Vec<VersionedTransaction>,
+    ) -> TransportResult<()> {
+        par_serialize_and_send_transaction_batch(&batch[..], self.tpu_addr())?;
+        Ok(())
     }
 }
 
-pub fn create_client(
-    (rpc, tpu): (SocketAddr, SocketAddr),
-    range: (u16, u16),
-) -> ThinClient<UdpTpuConnection> {
-    let (_, transactions_socket) =
-        solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), range).unwrap();
-    ThinClient::<UdpTpuConnection>::new(rpc, tpu, transactions_socket)
+pub fn create_client(rpc: SocketAddr, tpu: SocketAddr) -> ThinClient {
+    ThinClient::new(rpc, tpu)
 }
 
 pub fn create_client_with_timeout(
-    (rpc, tpu): (SocketAddr, SocketAddr),
-    range: (u16, u16),
+    rpc: SocketAddr,
+    tpu: SocketAddr,
     timeout: Duration,
-) -> ThinClient<UdpTpuConnection> {
-    let (_, transactions_socket) =
-        solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), range).unwrap();
-    ThinClient::<UdpTpuConnection>::new_socket_with_timeout(rpc, tpu, transactions_socket, timeout)
+) -> ThinClient {
+    ThinClient::new_socket_with_timeout(rpc, tpu, timeout)
 }
 
 #[cfg(test)]

client/src/tpu_client.rs

@@ -1,6 +1,7 @@
 use {
     crate::{
         client_error::ClientError,
+        connection_cache::send_wire_transaction_async,
         pubsub_client::{PubsubClient, PubsubClientError, PubsubClientSubscription},
         rpc_client::RpcClient,
         rpc_request::MAX_GET_SIGNATURE_STATUSES_QUERY_ITEMS,
@@ -17,6 +18,7 @@ use {
         signature::SignerError,
         signers::Signers,
         transaction::{Transaction, TransactionError},
+        transport::{Result as TransportResult, TransportError},
     },
     std::{
         collections::{HashMap, HashSet, VecDeque},
@@ -73,7 +75,7 @@ impl Default for TpuClientConfig {
 /// Client which sends transactions directly to the current leader's TPU port over UDP.
 /// The client uses RPC to determine the current leader and fetch node contact info
 pub struct TpuClient {
-    send_socket: UdpSocket,
+    _deprecated: UdpSocket, // TpuClient now uses the connection_cache to choose a send_socket
     fanout_slots: u64,
     leader_tpu_service: LeaderTpuService,
     exit: Arc<AtomicBool>,
@@ -85,25 +87,48 @@ impl TpuClient {
     /// size
     pub fn send_transaction(&self, transaction: &Transaction) -> bool {
         let wire_transaction = serialize(transaction).expect("serialization should succeed");
-        self.send_wire_transaction(&wire_transaction)
+        self.send_wire_transaction(wire_transaction)
     }
 
     /// Send a wire transaction to the current and upcoming leader TPUs according to fanout size
-    pub fn send_wire_transaction(&self, wire_transaction: &[u8]) -> bool {
-        let mut sent = false;
+    pub fn send_wire_transaction(&self, wire_transaction: Vec<u8>) -> bool {
+        self.try_send_wire_transaction(wire_transaction).is_ok()
+    }
+
+    /// Serialize and send transaction to the current and upcoming leader TPUs according to fanout
+    /// size
+    /// Returns the last error if all sends fail
+    pub fn try_send_transaction(&self, transaction: &Transaction) -> TransportResult<()> {
+        let wire_transaction = serialize(transaction).expect("serialization should succeed");
+        self.try_send_wire_transaction(wire_transaction)
+    }
+
+    /// Send a wire transaction to the current and upcoming leader TPUs according to fanout size
+    /// Returns the last error if all sends fail
+    fn try_send_wire_transaction(&self, wire_transaction: Vec<u8>) -> TransportResult<()> {
+        let mut last_error: Option<TransportError> = None;
+        let mut some_success = false;
+
         for tpu_address in self
             .leader_tpu_service
             .leader_tpu_sockets(self.fanout_slots)
         {
-            if self
-                .send_socket
-                .send_to(wire_transaction, tpu_address)
-                .is_ok()
-            {
-                sent = true;
+            let result = send_wire_transaction_async(wire_transaction.clone(), &tpu_address);
+            if let Err(err) = result {
+                last_error = Some(err);
+            } else {
+                some_success = true;
             }
         }
-        sent
+        if !some_success {
+            Err(if let Some(err) = last_error {
+                err
+            } else {
+                std::io::Error::new(std::io::ErrorKind::Other, "No sends attempted").into()
+            })
+        } else {
+            Ok(())
+        }
     }
 
     /// Create a new client that disconnects when dropped
@@ -117,7 +142,7 @@ impl TpuClient {
             LeaderTpuService::new(rpc_client.clone(), websocket_url, exit.clone())?;
 
         Ok(Self {
-            send_socket: UdpSocket::bind("0.0.0.0:0").unwrap(),
+            _deprecated: UdpSocket::bind("0.0.0.0:0").unwrap(),
             fanout_slots: config.fanout_slots.min(MAX_FANOUT_SLOTS).max(1),
             leader_tpu_service,
             exit,
@@ -266,6 +291,10 @@ impl TpuClient {
         }
         Err(TpuSenderError::Custom("Max retries exceeded".into()))
     }
+
+    pub fn rpc_client(&self) -> &RpcClient {
+        &self.rpc_client
+    }
 }
 
 impl Drop for TpuClient {

client/src/tpu_connection.rs

@@ -1,21 +1,79 @@
 use {
-    solana_sdk::{transaction::Transaction, transport::Result as TransportResult},
-    std::net::{SocketAddr, UdpSocket},
+    rayon::iter::{IntoParallelIterator, ParallelIterator},
+    solana_metrics::MovingStat,
+    solana_sdk::{transaction::VersionedTransaction, transport::Result as TransportResult},
+    std::{
+        net::{SocketAddr, UdpSocket},
+        sync::{atomic::AtomicU64, Arc},
+    },
 };
 
+#[derive(Default)]
+pub struct ClientStats {
+    pub total_connections: AtomicU64,
+    pub connection_reuse: AtomicU64,
+    pub connection_errors: AtomicU64,
+
+    // these will be the last values of these stats
+    pub congestion_events: MovingStat,
+    pub tx_streams_blocked_uni: MovingStat,
+    pub tx_data_blocked: MovingStat,
+    pub tx_acks: MovingStat,
+}
+
 pub trait TpuConnection {
-    fn new(client_socket: UdpSocket, tpu_addr: SocketAddr) -> Self
-    where
-        Self: Sized;
+    fn new(client_socket: UdpSocket, tpu_addr: SocketAddr) -> Self;
 
     fn tpu_addr(&self) -> &SocketAddr;
 
-    fn send_transaction(&self, tx: &Transaction) -> TransportResult<()> {
-        let data = bincode::serialize(tx).expect("serialize Transaction in send_transaction");
-        self.send_wire_transaction(data)
+    fn serialize_and_send_transaction(
+        &self,
+        transaction: &VersionedTransaction,
+        stats: &ClientStats,
+    ) -> TransportResult<()> {
+        let wire_transaction =
+            bincode::serialize(transaction).expect("serialize Transaction in send_batch");
+        self.send_wire_transaction(&wire_transaction, stats)
     }
 
-    fn send_wire_transaction(&self, data: Vec<u8>) -> TransportResult<()>;
+    fn send_wire_transaction<T>(
+        &self,
+        wire_transaction: T,
+        stats: &ClientStats,
+    ) -> TransportResult<()>
+    where
+        T: AsRef<[u8]>;
 
-    fn send_batch(&self, transactions: Vec<Transaction>) -> TransportResult<()>;
+    fn send_wire_transaction_async(
+        &self,
+        wire_transaction: Vec<u8>,
+        stats: Arc<ClientStats>,
+    ) -> TransportResult<()>;
+
+    fn par_serialize_and_send_transaction_batch(
+        &self,
+        transactions: &[VersionedTransaction],
+        stats: &ClientStats,
+    ) -> TransportResult<()> {
+        let buffers = transactions
+            .into_par_iter()
+            .map(|tx| bincode::serialize(&tx).expect("serialize Transaction in send_batch"))
+            .collect::<Vec<_>>();
+
+        self.send_wire_transaction_batch(&buffers, stats)
+    }
+
+    fn send_wire_transaction_batch<T>(
+        &self,
+        buffers: &[T],
+        stats: &ClientStats,
+    ) -> TransportResult<()>
+    where
+        T: AsRef<[u8]>;
+
+    fn send_wire_transaction_batch_async(
+        &self,
+        buffers: Vec<Vec<u8>>,
+        stats: Arc<ClientStats>,
+    ) -> TransportResult<()>;
 }

client/src/udp_client.rs

@@ -2,9 +2,14 @@
 //! an interface for sending transactions
 
 use {
-    crate::tpu_connection::TpuConnection,
-    solana_sdk::{transaction::Transaction, transport::Result as TransportResult},
-    std::net::{SocketAddr, UdpSocket},
+    crate::tpu_connection::{ClientStats, TpuConnection},
+    core::iter::repeat,
+    solana_sdk::transport::Result as TransportResult,
+    solana_streamer::sendmmsg::batch_send,
+    std::{
+        net::{SocketAddr, UdpSocket},
+        sync::Arc,
+    },
 };
 
 pub struct UdpTpuConnection {
@@ -24,19 +29,46 @@ impl TpuConnection for UdpTpuConnection {
         &self.addr
     }
 
-    fn send_wire_transaction(&self, data: Vec<u8>) -> TransportResult<()> {
-        self.socket.send_to(&data[..], self.addr)?;
+    fn send_wire_transaction<T>(
+        &self,
+        wire_transaction: T,
+        _stats: &ClientStats,
+    ) -> TransportResult<()>
+    where
+        T: AsRef<[u8]>,
+    {
+        self.socket.send_to(wire_transaction.as_ref(), self.addr)?;
         Ok(())
     }
 
-    fn send_batch(&self, transactions: Vec<Transaction>) -> TransportResult<()> {
-        transactions
-            .into_iter()
-            .map(|tx| bincode::serialize(&tx).expect("serialize Transaction in send_batch"))
-            .try_for_each(|buff| -> TransportResult<()> {
-                self.socket.send_to(&buff[..], self.addr)?;
+    fn send_wire_transaction_async(
+        &self,
+        wire_transaction: Vec<u8>,
+        _stats: Arc<ClientStats>,
+    ) -> TransportResult<()> {
+        self.socket.send_to(wire_transaction.as_ref(), self.addr)?;
         Ok(())
-            })?;
+    }
+
+    fn send_wire_transaction_batch<T>(
+        &self,
+        buffers: &[T],
+        _stats: &ClientStats,
+    ) -> TransportResult<()>
+    where
+        T: AsRef<[u8]>,
+    {
+        let pkts: Vec<_> = buffers.iter().zip(repeat(self.tpu_addr())).collect();
+        batch_send(&self.socket, &pkts)?;
+        Ok(())
+    }
+    fn send_wire_transaction_batch_async(
+        &self,
+        buffers: Vec<Vec<u8>>,
+        _stats: Arc<ClientStats>,
+    ) -> TransportResult<()> {
+        let pkts: Vec<_> = buffers.into_iter().zip(repeat(self.tpu_addr())).collect();
+        batch_send(&self.socket, &pkts)?;
         Ok(())
     }
 }

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.10.3"
+version = "1.11.0"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-core"
 readme = "../README.md"
@@ -21,42 +21,41 @@ bs58 = "0.4.0"
 chrono = { version = "0.4.11", features = ["serde"] }
 crossbeam-channel = "0.5"
 dashmap = { version = "4.0.2", features = ["rayon", "raw-api"] }
-etcd-client = { version = "0.8.4", features = ["tls"] }
+etcd-client = { version = "0.9.0", features = ["tls"] }
 fs_extra = "1.2.0"
 histogram = "0.6.9"
 itertools = "0.10.3"
 log = "0.4.14"
-lru = "0.7.3"
+lru = "0.7.5"
 rand = "0.7.0"
 rand_chacha = "0.2.2"
 rayon = "1.5.1"
 retain_mut = "0.1.7"
 serde = "1.0.136"
 serde_derive = "1.0.103"
-solana-address-lookup-table-program = { path = "../programs/address-lookup-table", version = "=1.10.3" }
-solana-bloom = { path = "../bloom", version = "=1.10.3" }
-solana-client = { path = "../client", version = "=1.10.3" }
-solana-entry = { path = "../entry", version = "=1.10.3" }
-solana-frozen-abi = { path = "../frozen-abi", version = "=1.10.3" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.10.3" }
-solana-geyser-plugin-manager = { path = "../geyser-plugin-manager", version = "=1.10.3" }
-solana-gossip = { path = "../gossip", version = "=1.10.3" }
-solana-ledger = { path = "../ledger", version = "=1.10.3" }
-solana-measure = { path = "../measure", version = "=1.10.3" }
-solana-metrics = { path = "../metrics", version = "=1.10.3" }
-solana-net-utils = { path = "../net-utils", version = "=1.10.3" }
-solana-perf = { path = "../perf", version = "=1.10.3" }
-solana-poh = { path = "../poh", version = "=1.10.3" }
-solana-program-runtime = { path = "../program-runtime", version = "=1.10.3" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.10.3" }
-solana-replica-lib = { path = "../replica-lib", version = "=1.10.3" }
-solana-rpc = { path = "../rpc", version = "=1.10.3" }
-solana-runtime = { path = "../runtime", version = "=1.10.3" }
-solana-sdk = { path = "../sdk", version = "=1.10.3" }
-solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.10.3" }
-solana-streamer = { path = "../streamer", version = "=1.10.3" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.3" }
-solana-vote-program = { path = "../programs/vote", version = "=1.10.3" }
+solana-address-lookup-table-program = { path = "../programs/address-lookup-table", version = "=1.11.0" }
+solana-bloom = { path = "../bloom", version = "=1.11.0" }
+solana-client = { path = "../client", version = "=1.11.0" }
+solana-entry = { path = "../entry", version = "=1.11.0" }
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.11.0" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.11.0" }
+solana-geyser-plugin-manager = { path = "../geyser-plugin-manager", version = "=1.11.0" }
+solana-gossip = { path = "../gossip", version = "=1.11.0" }
+solana-ledger = { path = "../ledger", version = "=1.11.0" }
+solana-measure = { path = "../measure", version = "=1.11.0" }
+solana-metrics = { path = "../metrics", version = "=1.11.0" }
+solana-net-utils = { path = "../net-utils", version = "=1.11.0" }
+solana-perf = { path = "../perf", version = "=1.11.0" }
+solana-poh = { path = "../poh", version = "=1.11.0" }
+solana-program-runtime = { path = "../program-runtime", version = "=1.11.0" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.11.0" }
+solana-rpc = { path = "../rpc", version = "=1.11.0" }
+solana-runtime = { path = "../runtime", version = "=1.11.0" }
+solana-sdk = { path = "../sdk", version = "=1.11.0" }
+solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.11.0" }
+solana-streamer = { path = "../streamer", version = "=1.11.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.11.0" }
+solana-vote-program = { path = "../programs/vote", version = "=1.11.0" }
 sys-info = "0.9.1"
 tempfile = "3.3.0"
 thiserror = "1.0"
@@ -69,10 +68,10 @@ raptorq = "1.6.5"
 reqwest = { version = "0.11.10", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 serde_json = "1.0.79"
 serial_test = "0.6.0"
-solana-logger = { path = "../logger", version = "=1.10.3" }
-solana-program-runtime = { path = "../program-runtime", version = "=1.10.3" }
-solana-stake-program = { path = "../programs/stake", version = "=1.10.3" }
-solana-version = { path = "../version", version = "=1.10.3" }
+solana-logger = { path = "../logger", version = "=1.11.0" }
+solana-program-runtime = { path = "../program-runtime", version = "=1.11.0" }
+solana-stake-program = { path = "../programs/stake", version = "=1.11.0" }
+solana-version = { path = "../version", version = "=1.11.0" }
 static_assertions = "1.1.0"
 systemstat = "0.1.10"
 

core/benches/cluster_nodes.rs

@@ -9,7 +9,12 @@ use {
         retransmit_stage::RetransmitStage,
     },
     solana_gossip::contact_info::ContactInfo,
-    solana_sdk::{clock::Slot, hash::hashv, pubkey::Pubkey, signature::Signature},
+    solana_ledger::{
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        shred::Shred,
+    },
+    solana_runtime::bank::Bank,
+    solana_sdk::pubkey::Pubkey,
     test::Bencher,
 };
 
@@ -26,87 +31,48 @@ fn make_cluster_nodes<R: Rng>(
 
 fn get_retransmit_peers_deterministic(
     cluster_nodes: &ClusterNodes<RetransmitStage>,
-    slot: &Slot,
+    shred: &mut Shred,
     slot_leader: &Pubkey,
+    root_bank: &Bank,
     num_simulated_shreds: usize,
 ) {
     for i in 0..num_simulated_shreds {
-        // see Shred::seed
-        let shred_seed = hashv(&[
-            &slot.to_le_bytes(),
-            &(i as u32).to_le_bytes(),
-            &slot_leader.to_bytes(),
-        ])
-        .to_bytes();
-
-        let (_neighbors, _children) = cluster_nodes.get_retransmit_peers_deterministic(
-            shred_seed,
-            solana_gossip::cluster_info::DATA_PLANE_FANOUT,
+        shred.common_header.index = i as u32;
+        let (_neighbors, _children) = cluster_nodes.get_retransmit_peers(
             *slot_leader,
-        );
-    }
-}
-
-fn get_retransmit_peers_compat(
-    cluster_nodes: &ClusterNodes<RetransmitStage>,
-    slot_leader: &Pubkey,
-    signatures: &[Signature],
-) {
-    for signature in signatures.iter() {
-        // see Shred::seed
-        let signature = signature.as_ref();
-        let offset = signature.len().checked_sub(32).unwrap();
-        let shred_seed = signature[offset..].try_into().unwrap();
-
-        let (_neighbors, _children) = cluster_nodes.get_retransmit_peers_compat(
-            shred_seed,
+            shred,
+            root_bank,
             solana_gossip::cluster_info::DATA_PLANE_FANOUT,
-            *slot_leader,
         );
     }
 }
 
 fn get_retransmit_peers_deterministic_wrapper(b: &mut Bencher, unstaked_ratio: Option<(u32, u32)>) {
     let mut rng = rand::thread_rng();
+    let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
+    let bank = Bank::new_for_benches(&genesis_config);
     let (nodes, cluster_nodes) = make_cluster_nodes(&mut rng, unstaked_ratio);
     let slot_leader = nodes[1..].choose(&mut rng).unwrap().id;
     let slot = rand::random::<u64>();
+    let mut shred = Shred::new_empty_data_shred();
+    shred.common_header.slot = slot;
     b.iter(|| {
         get_retransmit_peers_deterministic(
             &cluster_nodes,
-            &slot,
+            &mut shred,
             &slot_leader,
+            &bank,
             NUM_SIMULATED_SHREDS,
         )
     });
 }
 
-fn get_retransmit_peers_compat_wrapper(b: &mut Bencher, unstaked_ratio: Option<(u32, u32)>) {
-    let mut rng = rand::thread_rng();
-    let (nodes, cluster_nodes) = make_cluster_nodes(&mut rng, unstaked_ratio);
-    let slot_leader = nodes[1..].choose(&mut rng).unwrap().id;
-    let signatures: Vec<_> = std::iter::repeat_with(Signature::new_unique)
-        .take(NUM_SIMULATED_SHREDS)
-        .collect();
-    b.iter(|| get_retransmit_peers_compat(&cluster_nodes, &slot_leader, &signatures));
-}
-
 #[bench]
 fn bench_get_retransmit_peers_deterministic_unstaked_ratio_1_2(b: &mut Bencher) {
     get_retransmit_peers_deterministic_wrapper(b, Some((1, 2)));
 }
 
-#[bench]
-fn bench_get_retransmit_peers_compat_unstaked_ratio_1_2(b: &mut Bencher) {
-    get_retransmit_peers_compat_wrapper(b, Some((1, 2)));
-}
-
 #[bench]
 fn bench_get_retransmit_peers_deterministic_unstaked_ratio_1_32(b: &mut Bencher) {
     get_retransmit_peers_deterministic_wrapper(b, Some((1, 32)));
 }
-
-#[bench]
-fn bench_get_retransmit_peers_compat_unstaked_ratio_1_32(b: &mut Bencher) {
-    get_retransmit_peers_compat_wrapper(b, Some((1, 32)));
-}

core/benches/sigverify_stage.rs

@@ -159,7 +159,7 @@ fn bench_sigverify_stage(bencher: &mut Bencher) {
         for _ in 0..batches.len() {
             if let Some(batch) = batches.pop() {
                 sent_len += batch.packets.len();
-                packet_s.send(batch).unwrap();
+                packet_s.send(vec![batch]).unwrap();
             }
         }
         let mut received = 0;

core/src/accounts_hash_verifier.rs

@@ -1,28 +1,28 @@
 // Service to verify accounts hashes with other known validator nodes.
 //
-// Each interval, publish the snapshat hash which is the full accounts state
+// Each interval, publish the snapshot hash which is the full accounts state
 // hash on gossip. Monitor gossip for messages from validators in the `--known-validator`s
 // set and halt the node if a mismatch is detected.
 
 use {
-    crossbeam_channel::RecvTimeoutError,
-    rayon::ThreadPool,
     solana_gossip::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES},
     solana_measure::measure::Measure,
     solana_runtime::{
-        accounts_db::{self, AccountsDb},
-        accounts_hash::HashStats,
+        accounts_hash::{CalcAccountsHashConfig, HashStats},
         snapshot_config::SnapshotConfig,
         snapshot_package::{
-            AccountsPackage, AccountsPackageReceiver, PendingSnapshotPackage, SnapshotPackage,
+            AccountsPackage, PendingAccountsPackage, PendingSnapshotPackage, SnapshotPackage,
             SnapshotType,
         },
         sorted_storages::SortedStorages,
     },
-    solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey},
+    solana_sdk::{
+        clock::{Slot, SLOT_MS},
+        hash::Hash,
+        pubkey::Pubkey,
+    },
     std::{
         collections::{HashMap, HashSet},
-        path::{Path, PathBuf},
         sync::{
             atomic::{AtomicBool, Ordering},
             Arc,
@@ -38,7 +38,7 @@ pub struct AccountsHashVerifier {
 
 impl AccountsHashVerifier {
     pub fn new(
-        accounts_package_receiver: AccountsPackageReceiver,
+        pending_accounts_package: PendingAccountsPackage,
         pending_snapshot_package: Option<PendingSnapshotPackage>,
         exit: &Arc<AtomicBool>,
         cluster_info: &Arc<ClusterInfo>,
@@ -46,7 +46,6 @@ impl AccountsHashVerifier {
         halt_on_known_validators_accounts_hash_mismatch: bool,
         fault_injection_rate_slots: u64,
         snapshot_config: Option<SnapshotConfig>,
-        ledger_path: PathBuf,
     ) -> Self {
         let exit = exit.clone();
         let cluster_info = cluster_info.clone();
@@ -54,18 +53,17 @@ impl AccountsHashVerifier {
             .name("solana-hash-accounts".to_string())
             .spawn(move || {
                 let mut hashes = vec![];
-                let mut thread_pool = None;
                 loop {
                     if exit.load(Ordering::Relaxed) {
                         break;
                     }
 
-                    match accounts_package_receiver.recv_timeout(Duration::from_secs(1)) {
-                        Ok(accounts_package) => {
-                            if accounts_package.hash_for_testing.is_some() && thread_pool.is_none()
-                            {
-                                thread_pool = Some(accounts_db::make_min_priority_thread_pool());
+                    let accounts_package = pending_accounts_package.lock().unwrap().take();
+                    if accounts_package.is_none() {
+                        std::thread::sleep(Duration::from_millis(SLOT_MS));
+                        continue;
                     }
+                    let accounts_package = accounts_package.unwrap();
 
                     Self::process_accounts_package(
                         accounts_package,
@@ -77,14 +75,8 @@ impl AccountsHashVerifier {
                         &exit,
                         fault_injection_rate_slots,
                         snapshot_config.as_ref(),
-                                thread_pool.as_ref(),
-                                &ledger_path,
                     );
                 }
-                        Err(RecvTimeoutError::Disconnected) => break,
-                        Err(RecvTimeoutError::Timeout) => (),
-                    }
-                }
             })
             .unwrap();
         Self {
@@ -103,10 +95,8 @@ impl AccountsHashVerifier {
         exit: &Arc<AtomicBool>,
         fault_injection_rate_slots: u64,
         snapshot_config: Option<&SnapshotConfig>,
-        thread_pool: Option<&ThreadPool>,
-        ledger_path: &Path,
     ) {
-        Self::verify_accounts_package_hash(&accounts_package, thread_pool, ledger_path);
+        let accounts_hash = Self::calculate_and_verify_accounts_hash(&accounts_package);
 
         Self::push_accounts_hashes_to_cluster(
             &accounts_package,
@@ -116,39 +106,63 @@ impl AccountsHashVerifier {
             hashes,
             exit,
             fault_injection_rate_slots,
+            accounts_hash,
         );
 
-        Self::submit_for_packaging(accounts_package, pending_snapshot_package, snapshot_config);
+        Self::submit_for_packaging(
+            accounts_package,
+            pending_snapshot_package,
+            snapshot_config,
+            accounts_hash,
+        );
     }
 
-    fn verify_accounts_package_hash(
-        accounts_package: &AccountsPackage,
-        thread_pool: Option<&ThreadPool>,
-        ledger_path: &Path,
-    ) {
+    /// returns calculated accounts hash
+    fn calculate_and_verify_accounts_hash(accounts_package: &AccountsPackage) -> Hash {
         let mut measure_hash = Measure::start("hash");
-        if let Some(expected_hash) = accounts_package.hash_for_testing {
+        let mut sort_time = Measure::start("sort_storages");
         let sorted_storages = SortedStorages::new(&accounts_package.snapshot_storages);
-            let (hash, lamports) = AccountsDb::calculate_accounts_hash_without_index(
-                ledger_path,
+        sort_time.stop();
+
+        let mut timings = HashStats {
+            storage_sort_us: sort_time.as_us(),
+            ..HashStats::default()
+        };
+        timings.calc_storage_size_quartiles(&accounts_package.snapshot_storages);
+
+        let (accounts_hash, lamports) = accounts_package
+            .accounts
+            .accounts_db
+            .calculate_accounts_hash_without_index(
+                &CalcAccountsHashConfig {
+                    use_bg_thread_pool: true,
+                    check_hash: false,
+                    ancestors: None,
+                    use_write_cache: false,
+                    epoch_schedule: &accounts_package.epoch_schedule,
+                    rent_collector: &accounts_package.rent_collector,
+                },
                 &sorted_storages,
-                thread_pool,
-                HashStats::default(),
-                false,
-                None,
-                None, // this will fail with filler accounts
-                None, // this code path is only for testing, so use default # passes here
+                timings,
             )
             .unwrap();
 
         assert_eq!(accounts_package.expected_capitalization, lamports);
-            assert_eq!(expected_hash, hash);
+        if let Some(expected_hash) = accounts_package.accounts_hash_for_testing {
+            assert_eq!(expected_hash, accounts_hash);
         };
+
         measure_hash.stop();
+        solana_runtime::serde_snapshot::reserialize_bank_with_new_accounts_hash(
+            accounts_package.snapshot_links.path(),
+            accounts_package.slot,
+            &accounts_hash,
+        );
         datapoint_info!(
             "accounts_hash_verifier",
             ("calculate_hash", measure_hash.as_us(), i64),
         );
+        accounts_hash
     }
 
     fn push_accounts_hashes_to_cluster(
@@ -159,8 +173,8 @@ impl AccountsHashVerifier {
         hashes: &mut Vec<(Slot, Hash)>,
         exit: &Arc<AtomicBool>,
         fault_injection_rate_slots: u64,
+        accounts_hash: Hash,
     ) {
-        let hash = accounts_package.hash;
         if fault_injection_rate_slots != 0
             && accounts_package.slot % fault_injection_rate_slots == 0
         {
@@ -171,10 +185,10 @@ impl AccountsHashVerifier {
             };
             warn!("inserting fault at slot: {}", accounts_package.slot);
             let rand = thread_rng().gen_range(0, 10);
-            let hash = extend_and_hash(&hash, &[rand]);
+            let hash = extend_and_hash(&accounts_hash, &[rand]);
             hashes.push((accounts_package.slot, hash));
         } else {
-            hashes.push((accounts_package.slot, hash));
+            hashes.push((accounts_package.slot, accounts_hash));
         }
 
         while hashes.len() > MAX_SNAPSHOT_HASHES {
@@ -198,6 +212,7 @@ impl AccountsHashVerifier {
         accounts_package: AccountsPackage,
         pending_snapshot_package: Option<&PendingSnapshotPackage>,
         snapshot_config: Option<&SnapshotConfig>,
+        accounts_hash: Hash,
     ) {
         if accounts_package.snapshot_type.is_none()
             || pending_snapshot_package.is_none()
@@ -206,7 +221,7 @@ impl AccountsHashVerifier {
             return;
         };
 
-        let snapshot_package = SnapshotPackage::from(accounts_package);
+        let snapshot_package = SnapshotPackage::new(accounts_package, accounts_hash);
         let pending_snapshot_package = pending_snapshot_package.unwrap();
         let _snapshot_config = snapshot_config.unwrap();
 
@@ -284,13 +299,18 @@ mod tests {
     use {
         super::*,
         solana_gossip::{cluster_info::make_accounts_hashes_message, contact_info::ContactInfo},
-        solana_runtime::snapshot_utils::{ArchiveFormat, SnapshotVersion},
+        solana_runtime::{
+            rent_collector::RentCollector,
+            snapshot_utils::{ArchiveFormat, SnapshotVersion},
+        },
         solana_sdk::{
             genesis_config::ClusterType,
             hash::hash,
             signature::{Keypair, Signer},
+            sysvar::epoch_schedule::EpochSchedule,
         },
         solana_streamer::socket::SocketAddrSpace,
+        std::str::FromStr,
     };
 
     fn new_test_cluster_info(contact_info: ContactInfo) -> ClusterInfo {
@@ -353,6 +373,8 @@ mod tests {
             incremental_snapshot_archive_interval_slots: Slot::MAX,
             ..SnapshotConfig::default()
         };
+        let accounts = Arc::new(solana_runtime::accounts::Accounts::default_for_tests());
+        let expected_hash = Hash::from_str("GKot5hBsd81kMupNCXHaqbhv3huEbxAFMLnpcX2hniwn").unwrap();
         for i in 0..MAX_SNAPSHOT_HASHES + 1 {
             let accounts_package = AccountsPackage {
                 slot: full_snapshot_archive_interval_slots + i as u64,
@@ -360,18 +382,18 @@ mod tests {
                 slot_deltas: vec![],
                 snapshot_links: TempDir::new().unwrap(),
                 snapshot_storages: vec![],
-                hash: hash(&[i as u8]),
                 archive_format: ArchiveFormat::TarBzip2,
                 snapshot_version: SnapshotVersion::default(),
                 snapshot_archives_dir: PathBuf::default(),
                 expected_capitalization: 0,
-                hash_for_testing: None,
+                accounts_hash_for_testing: None,
                 cluster_type: ClusterType::MainnetBeta,
                 snapshot_type: None,
+                accounts: Arc::clone(&accounts),
+                epoch_schedule: EpochSchedule::default(),
+                rent_collector: RentCollector::default(),
             };
 
-            let ledger_path = TempDir::new().unwrap();
-
             AccountsHashVerifier::process_accounts_package(
                 accounts_package,
                 &cluster_info,
@@ -382,8 +404,6 @@ mod tests {
                 &exit,
                 0,
                 Some(&snapshot_config),
-                None,
-                ledger_path.path(),
             );
 
             // sleep for 1ms to create a newer timestmap for gossip entry
@@ -399,13 +419,13 @@ mod tests {
         assert_eq!(cluster_hashes.len(), MAX_SNAPSHOT_HASHES);
         assert_eq!(
             cluster_hashes[0],
-            (full_snapshot_archive_interval_slots + 1, hash(&[1]))
+            (full_snapshot_archive_interval_slots + 1, expected_hash)
         );
         assert_eq!(
             cluster_hashes[MAX_SNAPSHOT_HASHES - 1],
             (
                 full_snapshot_archive_interval_slots + MAX_SNAPSHOT_HASHES as u64,
-                hash(&[MAX_SNAPSHOT_HASHES as u8])
+                expected_hash
             )
         );
     }

core/src/banking_stage.rs

@@ -1,5 +1,5 @@
 //! The `banking_stage` processes Transaction messages. It is intended to be used
-//! to contruct a software pipeline. The stage uses all available CPU cores and
+//! to construct a software pipeline. The stage uses all available CPU cores and
 //! can do its processing in parallel with signature verification on the GPU.
 use {
     crate::{
@@ -14,6 +14,7 @@ use {
     histogram::Histogram,
     itertools::Itertools,
     retain_mut::RetainMut,
+    solana_client::connection_cache::send_wire_transaction_batch,
     solana_entry::entry::hash_transactions,
     solana_gossip::{cluster_info::ClusterInfo, contact_info::ContactInfo},
     solana_ledger::blockstore_processor::TransactionStatusSender,
@@ -22,7 +23,7 @@ use {
     solana_perf::{
         cuda_runtime::PinnedVec,
         data_budget::DataBudget,
-        packet::{limited_deserialize, Packet, PacketBatch, PACKETS_PER_BATCH},
+        packet::{Packet, PacketBatch, PACKETS_PER_BATCH},
         perf_libs,
     },
     solana_poh::poh_recorder::{BankStart, PohRecorder, PohRecorderError, TransactionRecorder},
@@ -44,15 +45,12 @@ use {
             MAX_TRANSACTION_FORWARDING_DELAY_GPU,
         },
         feature_set,
-        message::Message,
         pubkey::Pubkey,
         saturating_add_assign,
         timing::{duration_as_ms, timestamp, AtomicInterval},
-        transaction::{
-            self, AddressLoader, SanitizedTransaction, TransactionError, VersionedTransaction,
+        transaction::{self, AddressLoader, SanitizedTransaction, TransactionError},
+        transport::TransportError,
     },
-    },
-    solana_streamer::sendmmsg::{batch_send, SendPktsError},
     solana_transaction_status::token_balances::{
         collect_token_balances, TransactionTokenBalancesSet,
     },
@@ -60,7 +58,7 @@ use {
         cmp,
         collections::HashMap,
         env,
-        net::{SocketAddr, UdpSocket},
+        net::SocketAddr,
         sync::{
             atomic::{AtomicU64, AtomicUsize, Ordering},
             Arc, Mutex, RwLock,
@@ -194,7 +192,7 @@ impl BankingStageStats {
     }
 
     fn report(&mut self, report_interval_ms: u64) {
-        // skip repoting metrics if stats is empty
+        // skip reporting metrics if stats is empty
         if self.is_empty() {
             return;
         }
@@ -482,11 +480,10 @@ impl BankingStage {
     /// Forwards all valid, unprocessed packets in the buffer, up to a rate limit. Returns
     /// the number of successfully forwarded packets in second part of tuple
     fn forward_buffered_packets(
-        socket: &std::net::UdpSocket,
         tpu_forwards: &std::net::SocketAddr,
         packets: Vec<&Packet>,
         data_budget: &DataBudget,
-    ) -> (std::io::Result<()>, usize) {
+    ) -> (std::result::Result<(), TransportError>, usize) {
         const INTERVAL_MS: u64 = 100;
         const MAX_BYTES_PER_SECOND: usize = 10_000 * 1200;
         const MAX_BYTES_PER_INTERVAL: usize = MAX_BYTES_PER_SECOND * INTERVAL_MS as usize / 1000;
@@ -502,18 +499,35 @@ impl BankingStage {
             .iter()
             .filter_map(|p| {
                 if !p.meta.forwarded() && data_budget.take(p.meta.size) {
-                    Some((&p.data[..p.meta.size], tpu_forwards))
+                    Some(&p.data[..p.meta.size])
                 } else {
                     None
                 }
             })
             .collect();
 
+        // TODO: see https://github.com/solana-labs/solana/issues/23819
+        // fix this so returns the correct number of succeeded packets
+        // when there's an error sending the batch. This was left as-is for now
+        // in favor of shipping Quic support, which was considered higher-priority
         if !packet_vec.is_empty() {
             inc_new_counter_info!("banking_stage-forwarded_packets", packet_vec.len());
-            if let Err(SendPktsError::IoError(ioerr, num_failed)) = batch_send(socket, &packet_vec)
-            {
-                return (Err(ioerr), packet_vec.len().saturating_sub(num_failed));
+
+            let mut measure = Measure::start("banking_stage-forward-us");
+
+            let res = send_wire_transaction_batch(&packet_vec, tpu_forwards);
+
+            measure.stop();
+            inc_new_counter_info!(
+                "banking_stage-forward-us",
+                measure.as_us() as usize,
+                1000,
+                1000
+            );
+
+            if let Err(err) = res {
+                inc_new_counter_info!("banking_stage-forward_packets-failed-batches", 1);
+                return (Err(err), 0);
             }
         }
 
@@ -541,7 +555,6 @@ impl BankingStage {
         let mut reached_end_of_slot: Option<EndOfSlot> = None;
 
         RetainMut::retain_mut(buffered_packet_batches, |deserialized_packet_batch| {
-            let packet_batch = &deserialized_packet_batch.packet_batch;
             let original_unprocessed_indexes = deserialized_packet_batch
                 .unprocessed_packets
                 .keys()
@@ -555,8 +568,7 @@ impl BankingStage {
                 let should_retain = if let Some(bank) = &end_of_slot.working_bank {
                     let new_unprocessed_indexes = Self::filter_unprocessed_packets_at_end_of_slot(
                         bank,
-                        packet_batch,
-                        &original_unprocessed_indexes,
+                        &deserialized_packet_batch.unprocessed_packets,
                         my_pubkey,
                         end_of_slot.next_slot_leader,
                         banking_stage_stats,
@@ -607,8 +619,7 @@ impl BankingStage {
                                     &working_bank,
                                     &bank_creation_time,
                                     recorder,
-                                    packet_batch,
-                                    original_unprocessed_indexes.to_owned(),
+                                    &deserialized_packet_batch.unprocessed_packets,
                                     transaction_status_sender.clone(),
                                     gossip_vote_sender,
                                     banking_stage_stats,
@@ -700,14 +711,11 @@ impl BankingStage {
 
                     // `original_unprocessed_indexes` must have remaining packets to process
                     // if not yet processed.
-                    assert!(Self::packet_has_more_unprocessed_transactions(
-                        &original_unprocessed_indexes
-                    ));
+                    assert!(!original_unprocessed_indexes.is_empty());
                     true
                 }
             }
         });
-
         proc_start.stop();
 
         debug!(
@@ -766,7 +774,6 @@ impl BankingStage {
     #[allow(clippy::too_many_arguments)]
     fn process_buffered_packets(
         my_pubkey: &Pubkey,
-        socket: &std::net::UdpSocket,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
         cluster_info: &ClusterInfo,
         buffered_packet_batches: &mut UnprocessedPacketBatches,
@@ -846,7 +853,6 @@ impl BankingStage {
                             cluster_info,
                             buffered_packet_batches,
                             poh_recorder,
-                            socket,
                             false,
                             data_budget,
                             slot_metrics_tracker,
@@ -865,7 +871,6 @@ impl BankingStage {
                             cluster_info,
                             buffered_packet_batches,
                             poh_recorder,
-                            socket,
                             true,
                             data_budget,
                             slot_metrics_tracker,
@@ -887,7 +892,6 @@ impl BankingStage {
         cluster_info: &ClusterInfo,
         buffered_packet_batches: &mut UnprocessedPacketBatches,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
-        socket: &UdpSocket,
         hold: bool,
         data_budget: &DataBudget,
         slot_metrics_tracker: &mut LeaderSlotMetricsTracker,
@@ -913,7 +917,7 @@ impl BankingStage {
             Self::filter_valid_packets_for_forwarding(buffered_packet_batches.iter());
         let forwardable_packets_len = forwardable_packets.len();
         let (_forward_result, sucessful_forwarded_packets_count) =
-            Self::forward_buffered_packets(socket, &addr, forwardable_packets, data_budget);
+            Self::forward_buffered_packets(&addr, forwardable_packets, data_budget);
         let failed_forwarded_packets_count =
             forwardable_packets_len.saturating_sub(sucessful_forwarded_packets_count);
 
@@ -958,7 +962,6 @@ impl BankingStage {
         cost_model: Arc<RwLock<CostModel>>,
     ) {
         let recorder = poh_recorder.lock().unwrap().recorder();
-        let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
         let mut buffered_packet_batches = UnprocessedPacketBatches::with_capacity(batch_limit);
         let mut banking_stage_stats = BankingStageStats::new(id);
         let qos_service = QosService::new(cost_model, id);
@@ -970,7 +973,6 @@ impl BankingStage {
                     |_| {
                         Self::process_buffered_packets(
                             &my_pubkey,
-                            &socket,
                             poh_recorder,
                             cluster_info,
                             &mut buffered_packet_batches,
@@ -1183,6 +1185,7 @@ impl BankingStage {
                     MAX_PROCESSING_AGE,
                     transaction_status_sender.is_some(),
                     transaction_status_sender.is_some(),
+                    transaction_status_sender.is_some(),
                     &mut execute_and_commit_timings.execute_timings,
                 )
             },
@@ -1342,38 +1345,28 @@ impl BankingStage {
         gossip_vote_sender: &ReplayVoteSender,
         qos_service: &QosService,
     ) -> ProcessTransactionBatchOutput {
-        let ((transactions_qos_results, cost_model_throttled_transactions_count), cost_model_time) =
-            Measure::this(
-                |_| {
-                    let tx_costs = qos_service.compute_transaction_costs(txs.iter());
+        let mut cost_model_time = Measure::start("cost_model");
+
+        let transaction_costs = qos_service.compute_transaction_costs(txs.iter());
 
         let (transactions_qos_results, num_included) =
-                        qos_service.select_transactions_per_cost(txs.iter(), tx_costs.iter(), bank);
+            qos_service.select_transactions_per_cost(txs.iter(), transaction_costs.iter(), bank);
 
-                    let cost_model_throttled_transactions_count =
-                        txs.len().saturating_sub(num_included);
+        let cost_model_throttled_transactions_count = txs.len().saturating_sub(num_included);
 
         qos_service.accumulate_estimated_transaction_costs(
             &Self::accumulate_batched_transaction_costs(
-                            tx_costs.iter(),
+                transaction_costs.iter(),
                 transactions_qos_results.iter(),
             ),
         );
-                    (
-                        transactions_qos_results,
-                        cost_model_throttled_transactions_count,
-                    )
-                },
-                (),
-                "cost_model",
-            );
+        cost_model_time.stop();
 
         // Only lock accounts for those transactions are selected for the block;
         // Once accounts are locked, other threads cannot encode transactions that will modify the
         // same account state
         let mut lock_time = Measure::start("lock_time");
-        let batch =
-            bank.prepare_sanitized_batch_with_results(txs, transactions_qos_results.into_iter());
+        let batch = bank.prepare_sanitized_batch_with_results(txs, transactions_qos_results.iter());
         lock_time.stop();
 
         // retryable_txs includes AccountInUse, WouldExceedMaxBlockCostLimit
@@ -1388,21 +1381,31 @@ impl BankingStage {
                 gossip_vote_sender,
             );
 
+        let mut unlock_time = Measure::start("unlock_time");
+        // Once the accounts are new transactions can enter the pipeline to process them
+        drop(batch);
+        unlock_time.stop();
+
         let ExecuteAndCommitTransactionsOutput {
             ref mut retryable_transaction_indexes,
             ref execute_and_commit_timings,
             ..
         } = execute_and_commit_transactions_output;
 
+        // TODO: This does not revert the cost tracker changes from all unexecuted transactions
+        // yet: For example tx that are too old will not be included in the block, but are not
+        // retryable.
+        QosService::update_or_remove_transaction_costs(
+            transaction_costs.iter(),
+            transactions_qos_results.iter(),
+            retryable_transaction_indexes,
+            bank,
+        );
+
         retryable_transaction_indexes
             .iter_mut()
             .for_each(|x| *x += chunk_offset);
 
-        let mut unlock_time = Measure::start("unlock_time");
-        // Once the accounts are new transactions can enter the pipeline to process them
-        drop(batch);
-        unlock_time.stop();
-
         let (cu, us) =
             Self::accumulate_execute_units_and_time(&execute_and_commit_timings.execute_timings);
         qos_service.accumulate_actual_execute_cu(cu);
@@ -1676,32 +1679,27 @@ impl BankingStage {
     // with their packet indexes.
     #[allow(clippy::needless_collect)]
     fn transactions_from_packets(
-        packet_batch: &PacketBatch,
-        transaction_indexes: &[usize],
+        deserialized_packet_batch: &HashMap<usize, DeserializedPacket>,
         feature_set: &Arc<feature_set::FeatureSet>,
         votes_only: bool,
         address_loader: impl AddressLoader,
     ) -> (Vec<SanitizedTransaction>, Vec<usize>) {
-        transaction_indexes
+        deserialized_packet_batch
             .iter()
-            .filter_map(|tx_index| {
-                let p = &packet_batch.packets[*tx_index];
-                if votes_only && !p.meta.is_simple_vote_tx() {
+            .filter_map(|(&tx_index, deserialized_packet)| {
+                if votes_only && !deserialized_packet.is_simple_vote {
                     return None;
                 }
 
-                let tx: VersionedTransaction = limited_deserialize(&p.data[0..p.meta.size]).ok()?;
-                let message_bytes = DeserializedPacketBatch::packet_message(p)?;
-                let message_hash = Message::hash_raw_message(message_bytes);
                 let tx = SanitizedTransaction::try_create(
-                    tx,
-                    message_hash,
-                    Some(p.meta.is_simple_vote_tx()),
+                    deserialized_packet.versioned_transaction.clone(),
+                    deserialized_packet.message_hash,
+                    Some(deserialized_packet.is_simple_vote),
                     address_loader.clone(),
                 )
                 .ok()?;
                 tx.verify_precompiles(feature_set).ok()?;
-                Some((tx, *tx_index))
+                Some((tx, tx_index))
             })
             .unzip()
     }
@@ -1750,8 +1748,7 @@ impl BankingStage {
         bank: &Arc<Bank>,
         bank_creation_time: &Instant,
         poh: &TransactionRecorder,
-        packet_batch: &PacketBatch,
-        packet_indexes: Vec<usize>,
+        deserialized_packet_batch: &HashMap<usize, DeserializedPacket>,
         transaction_status_sender: Option<TransactionStatusSender>,
         gossip_vote_sender: &ReplayVoteSender,
         banking_stage_stats: &BankingStageStats,
@@ -1762,8 +1759,7 @@ impl BankingStage {
         let ((transactions, transaction_to_packet_indexes), packet_conversion_time) = Measure::this(
             |_| {
                 Self::transactions_from_packets(
-                    packet_batch,
-                    &packet_indexes,
+                    deserialized_packet_batch,
                     &bank.feature_set,
                     bank.vote_only_bank(),
                     bank.as_ref(),
@@ -1851,8 +1847,7 @@ impl BankingStage {
 
     fn filter_unprocessed_packets_at_end_of_slot(
         bank: &Arc<Bank>,
-        packet_batch: &PacketBatch,
-        transaction_indexes: &[usize],
+        deserialized_packet_batch: &HashMap<usize, DeserializedPacket>,
         my_pubkey: &Pubkey,
         next_leader: Option<Pubkey>,
         banking_stage_stats: &BankingStageStats,
@@ -1862,15 +1857,17 @@ impl BankingStage {
         // Filtering helps if we were going to forward the packets to some other node
         if let Some(leader) = next_leader {
             if leader == *my_pubkey {
-                return transaction_indexes.to_vec();
+                return deserialized_packet_batch
+                    .keys()
+                    .cloned()
+                    .collect::<Vec<usize>>();
             }
         }
 
         let mut unprocessed_packet_conversion_time =
             Measure::start("unprocessed_packet_conversion");
         let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets(
-            packet_batch,
-            transaction_indexes,
+            deserialized_packet_batch,
             &bank.feature_set,
             bank.vote_only_bank(),
             bank.as_ref(),
@@ -2015,7 +2012,7 @@ impl BankingStage {
         banking_stage_stats: &mut BankingStageStats,
         slot_metrics_tracker: &mut LeaderSlotMetricsTracker,
     ) {
-        if Self::packet_has_more_unprocessed_transactions(&packet_indexes) {
+        if !packet_indexes.is_empty() {
             if unprocessed_packet_batches.len() >= batch_limit {
                 *dropped_packet_batches_count += 1;
                 if let Some(dropped_batch) = unprocessed_packet_batches.pop_front() {
@@ -2041,10 +2038,6 @@ impl BankingStage {
         }
     }
 
-    fn packet_has_more_unprocessed_transactions(packet_indexes: &[usize]) -> bool {
-        !packet_indexes.is_empty()
-    }
-
     pub fn join(self) -> thread::Result<()> {
         for bank_thread_hdl in self.bank_thread_hdls {
             bank_thread_hdl.join()?;
@@ -2108,7 +2101,7 @@ mod tests {
             get_tmp_ledger_path_auto_delete,
             leader_schedule_cache::LeaderScheduleCache,
         },
-        solana_perf::packet::{to_packet_batches, PacketFlags},
+        solana_perf::packet::{limited_deserialize, to_packet_batches, PacketFlags},
         solana_poh::{
             poh_recorder::{create_test_recorder, Record, WorkingBankEntry},
             poh_service::PohService,
@@ -2128,7 +2121,10 @@ mod tests {
             signature::{Keypair, Signer},
             system_instruction::SystemError,
             system_transaction,
-            transaction::{MessageHash, SimpleAddressLoader, Transaction, TransactionError},
+            transaction::{
+                MessageHash, SimpleAddressLoader, Transaction, TransactionError,
+                VersionedTransaction,
+            },
         },
         solana_streamer::{recvmmsg::recv_mmsg, socket::SocketAddrSpace},
         solana_transaction_status::{TransactionStatusMeta, VersionedTransactionWithStatusMeta},
@@ -2156,6 +2152,7 @@ mod tests {
             log_messages: None,
             inner_instructions: None,
             durable_nonce_fee: None,
+            return_data: None,
         })
     }
 
@@ -2887,6 +2884,131 @@ mod tests {
         Blockstore::destroy(ledger_path.path()).unwrap();
     }
 
+    #[test]
+    fn test_bank_process_and_record_transactions_cost_tracker() {
+        solana_logger::setup();
+        let GenesisConfigInfo {
+            genesis_config,
+            mint_keypair,
+            ..
+        } = create_slow_genesis_config(10_000);
+        let bank = Arc::new(Bank::new_no_wallclock_throttle_for_tests(&genesis_config));
+        let pubkey = solana_sdk::pubkey::new_rand();
+
+        let ledger_path = get_tmp_ledger_path_auto_delete!();
+        {
+            let blockstore = Blockstore::open(ledger_path.path())
+                .expect("Expected to be able to open database ledger");
+            let (poh_recorder, _entry_receiver, record_receiver) = PohRecorder::new(
+                bank.tick_height(),
+                bank.last_blockhash(),
+                bank.clone(),
+                Some((4, 4)),
+                bank.ticks_per_slot(),
+                &pubkey,
+                &Arc::new(blockstore),
+                &Arc::new(LeaderScheduleCache::new_from_bank(&bank)),
+                &Arc::new(PohConfig::default()),
+                Arc::new(AtomicBool::default()),
+            );
+            let recorder = poh_recorder.recorder();
+            let poh_recorder = Arc::new(Mutex::new(poh_recorder));
+
+            let poh_simulator = simulate_poh(record_receiver, &poh_recorder);
+
+            poh_recorder.lock().unwrap().set_bank(&bank);
+            let (gossip_vote_sender, _gossip_vote_receiver) = unbounded();
+
+            let qos_service = QosService::new(Arc::new(RwLock::new(CostModel::default())), 1);
+
+            let get_block_cost = || bank.read_cost_tracker().unwrap().block_cost();
+            let get_tx_count = || bank.read_cost_tracker().unwrap().transaction_count();
+            assert_eq!(get_block_cost(), 0);
+            assert_eq!(get_tx_count(), 0);
+
+            //
+            // TEST: cost tracker's block cost increases when successfully processing a tx
+            //
+
+            let transactions = sanitize_transactions(vec![system_transaction::transfer(
+                &mint_keypair,
+                &pubkey,
+                1,
+                genesis_config.hash(),
+            )]);
+
+            let process_transactions_batch_output = BankingStage::process_and_record_transactions(
+                &bank,
+                &transactions,
+                &recorder,
+                0,
+                None,
+                &gossip_vote_sender,
+                &qos_service,
+            );
+
+            let ExecuteAndCommitTransactionsOutput {
+                executed_with_successful_result_count,
+                commit_transactions_result,
+                ..
+            } = process_transactions_batch_output.execute_and_commit_transactions_output;
+            assert_eq!(executed_with_successful_result_count, 1);
+            assert!(commit_transactions_result.is_ok());
+
+            let single_transfer_cost = get_block_cost();
+            assert_ne!(single_transfer_cost, 0);
+            assert_eq!(get_tx_count(), 1);
+
+            //
+            // TEST: When a tx in a batch can't be executed (here because of account
+            // locks), then its cost does not affect the cost tracker.
+            //
+
+            let allocate_keypair = Keypair::new();
+            let transactions = sanitize_transactions(vec![
+                system_transaction::transfer(&mint_keypair, &pubkey, 2, genesis_config.hash()),
+                // intentionally use a tx that has a different cost
+                system_transaction::allocate(
+                    &mint_keypair,
+                    &allocate_keypair,
+                    genesis_config.hash(),
+                    1,
+                ),
+            ]);
+
+            let process_transactions_batch_output = BankingStage::process_and_record_transactions(
+                &bank,
+                &transactions,
+                &recorder,
+                0,
+                None,
+                &gossip_vote_sender,
+                &qos_service,
+            );
+
+            let ExecuteAndCommitTransactionsOutput {
+                executed_with_successful_result_count,
+                commit_transactions_result,
+                retryable_transaction_indexes,
+                ..
+            } = process_transactions_batch_output.execute_and_commit_transactions_output;
+            assert_eq!(executed_with_successful_result_count, 1);
+            assert!(commit_transactions_result.is_ok());
+            assert_eq!(retryable_transaction_indexes, vec![1]);
+
+            assert_eq!(get_block_cost(), 2 * single_transfer_cost);
+            assert_eq!(get_tx_count(), 2);
+
+            poh_recorder
+                .lock()
+                .unwrap()
+                .is_exited
+                .store(true, Ordering::Relaxed);
+            let _ = poh_simulator.join();
+        }
+        Blockstore::destroy(ledger_path.path()).unwrap();
+    }
+
     fn simulate_poh(
         record_receiver: CrossbeamReceiver<Record>,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
@@ -3835,7 +3957,6 @@ mod tests {
 
             let local_node = Node::new_localhost_with_pubkey(validator_pubkey);
             let cluster_info = new_test_cluster_info(local_node.info);
-            let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
             let recv_socket = &local_node.sockets.tpu_forwards[0];
 
             let test_cases = vec![
@@ -3857,7 +3978,6 @@ mod tests {
                     &cluster_info,
                     &mut unprocessed_packet_batches,
                     &poh_recorder,
-                    &send_socket,
                     true,
                     &data_budget,
                     &mut LeaderSlotMetricsTracker::new(0),
@@ -3935,7 +4055,6 @@ mod tests {
 
             let local_node = Node::new_localhost_with_pubkey(validator_pubkey);
             let cluster_info = new_test_cluster_info(local_node.info);
-            let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
             let recv_socket = &local_node.sockets.tpu_forwards[0];
 
             let test_cases = vec![
@@ -3969,7 +4088,6 @@ mod tests {
                     &cluster_info,
                     &mut unprocessed_packet_batches,
                     &poh_recorder,
-                    &send_socket,
                     hold,
                     &DataBudget::default(),
                     &mut LeaderSlotMetricsTracker::new(0),
@@ -4113,7 +4231,7 @@ mod tests {
     fn make_test_packets(
         transactions: Vec<Transaction>,
         vote_indexes: Vec<usize>,
-    ) -> (PacketBatch, Vec<usize>) {
+    ) -> DeserializedPacketBatch {
         let capacity = transactions.len();
         let mut packet_batch = PacketBatch::with_capacity(capacity);
         let mut packet_indexes = Vec::with_capacity(capacity);
@@ -4125,7 +4243,7 @@ mod tests {
         for index in vote_indexes.iter() {
             packet_batch.packets[*index].meta.flags |= PacketFlags::SIMPLE_VOTE_TX;
         }
-        (packet_batch, packet_indexes)
+        DeserializedPacketBatch::new(packet_batch, packet_indexes, false)
     }
 
     #[test]
@@ -4143,28 +4261,30 @@ mod tests {
             &keypair,
             None,
         );
+        let sorted = |mut v: Vec<usize>| {
+            v.sort_unstable();
+            v
+        };
 
         // packets with no votes
         {
             let vote_indexes = vec![];
-            let (packet_batch, packet_indexes) =
+            let deserialized_packet_batch =
                 make_test_packets(vec![transfer_tx.clone(), transfer_tx.clone()], vote_indexes);
 
             let mut votes_only = false;
             let (txs, tx_packet_index) = BankingStage::transactions_from_packets(
-                &packet_batch,
-                &packet_indexes,
+                &deserialized_packet_batch.unprocessed_packets,
                 &Arc::new(FeatureSet::default()),
                 votes_only,
                 SimpleAddressLoader::Disabled,
             );
             assert_eq!(2, txs.len());
-            assert_eq!(vec![0, 1], tx_packet_index);
+            assert_eq!(vec![0, 1], sorted(tx_packet_index));
 
             votes_only = true;
             let (txs, tx_packet_index) = BankingStage::transactions_from_packets(
-                &packet_batch,
-                &packet_indexes,
+                &deserialized_packet_batch.unprocessed_packets,
                 &Arc::new(FeatureSet::default()),
                 votes_only,
                 SimpleAddressLoader::Disabled,
@@ -4176,63 +4296,59 @@ mod tests {
         // packets with some votes
         {
             let vote_indexes = vec![0, 2];
-            let (packet_batch, packet_indexes) = make_test_packets(
+            let deserialized_packet_batch = make_test_packets(
                 vec![vote_tx.clone(), transfer_tx, vote_tx.clone()],
                 vote_indexes,
             );
 
             let mut votes_only = false;
             let (txs, tx_packet_index) = BankingStage::transactions_from_packets(
-                &packet_batch,
-                &packet_indexes,
+                &deserialized_packet_batch.unprocessed_packets,
                 &Arc::new(FeatureSet::default()),
                 votes_only,
                 SimpleAddressLoader::Disabled,
             );
             assert_eq!(3, txs.len());
-            assert_eq!(vec![0, 1, 2], tx_packet_index);
+            assert_eq!(vec![0, 1, 2], sorted(tx_packet_index));
 
             votes_only = true;
             let (txs, tx_packet_index) = BankingStage::transactions_from_packets(
-                &packet_batch,
-                &packet_indexes,
+                &deserialized_packet_batch.unprocessed_packets,
                 &Arc::new(FeatureSet::default()),
                 votes_only,
                 SimpleAddressLoader::Disabled,
             );
             assert_eq!(2, txs.len());
-            assert_eq!(vec![0, 2], tx_packet_index);
+            assert_eq!(vec![0, 2], sorted(tx_packet_index));
         }
 
         // packets with all votes
         {
             let vote_indexes = vec![0, 1, 2];
-            let (packet_batch, packet_indexes) = make_test_packets(
+            let deserialized_packet_batch = make_test_packets(
                 vec![vote_tx.clone(), vote_tx.clone(), vote_tx],
                 vote_indexes,
             );
 
             let mut votes_only = false;
             let (txs, tx_packet_index) = BankingStage::transactions_from_packets(
-                &packet_batch,
-                &packet_indexes,
+                &deserialized_packet_batch.unprocessed_packets,
                 &Arc::new(FeatureSet::default()),
                 votes_only,
                 SimpleAddressLoader::Disabled,
             );
             assert_eq!(3, txs.len());
-            assert_eq!(vec![0, 1, 2], tx_packet_index);
+            assert_eq!(vec![0, 1, 2], sorted(tx_packet_index));
 
             votes_only = true;
             let (txs, tx_packet_index) = BankingStage::transactions_from_packets(
-                &packet_batch,
-                &packet_indexes,
+                &deserialized_packet_batch.unprocessed_packets,
                 &Arc::new(FeatureSet::default()),
                 votes_only,
                 SimpleAddressLoader::Disabled,
             );
             assert_eq!(3, txs.len());
-            assert_eq!(vec![0, 1, 2], tx_packet_index);
+            assert_eq!(vec![0, 1, 2], sorted(tx_packet_index));
         }
     }
 

core/src/cluster_nodes.rs

@@ -10,13 +10,12 @@ use {
         crds::GossipRoute,
         crds_gossip_pull::CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS,
         crds_value::{CrdsData, CrdsValue},
-        weighted_shuffle::{weighted_best, weighted_shuffle, WeightedShuffle},
+        weighted_shuffle::WeightedShuffle,
     },
     solana_ledger::shred::Shred,
     solana_runtime::bank::Bank,
     solana_sdk::{
         clock::{Epoch, Slot},
-        feature_set,
         pubkey::Pubkey,
         signature::Keypair,
         timing::timestamp,
@@ -56,10 +55,6 @@ pub struct ClusterNodes<T> {
     // Reverse index from nodes pubkey to their index in self.nodes.
     index: HashMap<Pubkey, /*index:*/ usize>,
     weighted_shuffle: WeightedShuffle</*stake:*/ u64>,
-    // Weights and indices for sampling peers. weighted_{shuffle,best} expect
-    // weights >= 1. For backward compatibility we use max(1, stake) for
-    // weights and exclude nodes with no contact-info.
-    compat_index: Vec<(/*weight:*/ u64, /*index:*/ usize)>,
     _phantom: PhantomData<T>,
 }
 
@@ -92,14 +87,15 @@ impl Node {
 
 impl<T> ClusterNodes<T> {
     pub(crate) fn num_peers(&self) -> usize {
-        self.compat_index.len()
+        self.nodes.len().saturating_sub(1)
     }
 
     // A peer is considered live if they generated their contact info recently.
     pub(crate) fn num_peers_live(&self, now: u64) -> usize {
-        self.compat_index
+        self.nodes
             .iter()
-            .filter_map(|(_, index)| self.nodes[*index].contact_info())
+            .filter(|node| node.pubkey() != self.pubkey)
+            .filter_map(|node| node.contact_info())
             .filter(|node| {
                 let elapsed = if node.wallclock < now {
                     now - node.wallclock
@@ -120,20 +116,12 @@ impl ClusterNodes<BroadcastStage> {
     pub(crate) fn get_broadcast_addrs(
         &self,
         shred: &Shred,
-        root_bank: &Bank,
+        _root_bank: &Bank,
         fanout: usize,
         socket_addr_space: &SocketAddrSpace,
     ) -> Vec<SocketAddr> {
         const MAX_CONTACT_INFO_AGE: Duration = Duration::from_secs(2 * 60);
-        let shred_seed = shred.seed(self.pubkey, root_bank);
-        if !enable_turbine_peers_shuffle_patch(shred.slot(), root_bank) {
-            if let Some(node) = self.get_broadcast_peer(shred_seed) {
-                if socket_addr_space.check(&node.tvu) {
-                    return vec![node.tvu];
-                }
-            }
-            return Vec::default();
-        }
+        let shred_seed = shred.seed(self.pubkey);
         let mut rng = ChaChaRng::from_seed(shred_seed);
         let index = match self.weighted_shuffle.first(&mut rng) {
             None => return Vec::default(),
@@ -175,20 +163,6 @@ impl ClusterNodes<BroadcastStage> {
             .filter(|addr| ContactInfo::is_valid_address(addr, socket_addr_space))
             .collect()
     }
-
-    /// Returns the root of turbine broadcast tree, which the leader sends the
-    /// shred to.
-    fn get_broadcast_peer(&self, shred_seed: [u8; 32]) -> Option<&ContactInfo> {
-        if self.compat_index.is_empty() {
-            None
-        } else {
-            let index = weighted_best(&self.compat_index, shred_seed);
-            match &self.nodes[index].node {
-                NodeId::ContactInfo(node) => Some(node),
-                NodeId::Pubkey(_) => panic!("this should not happen!"),
-            }
-        }
-    }
 }
 
 impl ClusterNodes<RetransmitStage> {
@@ -223,32 +197,17 @@ impl ClusterNodes<RetransmitStage> {
             .collect()
     }
 
-    fn get_retransmit_peers(
+    pub fn get_retransmit_peers(
         &self,
         slot_leader: Pubkey,
         shred: &Shred,
-        root_bank: &Bank,
+        _root_bank: &Bank,
         fanout: usize,
     ) -> (
         Vec<&Node>, // neighbors
         Vec<&Node>, // children
     ) {
-        let shred_seed = shred.seed(slot_leader, root_bank);
-        if !enable_turbine_peers_shuffle_patch(shred.slot(), root_bank) {
-            return self.get_retransmit_peers_compat(shred_seed, fanout, slot_leader);
-        }
-        self.get_retransmit_peers_deterministic(shred_seed, fanout, slot_leader)
-    }
-
-    pub fn get_retransmit_peers_deterministic(
-        &self,
-        shred_seed: [u8; 32],
-        fanout: usize,
-        slot_leader: Pubkey,
-    ) -> (
-        Vec<&Node>, // neighbors
-        Vec<&Node>, // children
-    ) {
+        let shred_seed = shred.seed(slot_leader);
         let mut weighted_shuffle = self.weighted_shuffle.clone();
         // Exclude slot leader from list of nodes.
         if slot_leader == self.pubkey {
@@ -271,46 +230,6 @@ impl ClusterNodes<RetransmitStage> {
         debug_assert_eq!(neighbors[self_index % fanout].pubkey(), self.pubkey);
         (neighbors, children)
     }
-
-    pub fn get_retransmit_peers_compat(
-        &self,
-        shred_seed: [u8; 32],
-        fanout: usize,
-        slot_leader: Pubkey,
-    ) -> (
-        Vec<&Node>, // neighbors
-        Vec<&Node>, // children
-    ) {
-        // Exclude leader from list of nodes.
-        let (weights, index): (Vec<u64>, Vec<usize>) = if slot_leader == self.pubkey {
-            error!("retransmit from slot leader: {}", slot_leader);
-            self.compat_index.iter().copied().unzip()
-        } else {
-            self.compat_index
-                .iter()
-                .filter(|(_, i)| self.nodes[*i].pubkey() != slot_leader)
-                .copied()
-                .unzip()
-        };
-        let index: Vec<_> = {
-            let shuffle = weighted_shuffle(weights.into_iter(), shred_seed);
-            shuffle.into_iter().map(|i| index[i]).collect()
-        };
-        let self_index = index
-            .iter()
-            .position(|i| self.nodes[*i].pubkey() == self.pubkey)
-            .unwrap();
-        let (neighbors, children) = compute_retransmit_peers(fanout, self_index, &index);
-        // Assert that the node itself is included in the set of neighbors, at
-        // the right offset.
-        debug_assert_eq!(
-            self.nodes[neighbors[self_index % fanout]].pubkey(),
-            self.pubkey
-        );
-        let neighbors = neighbors.into_iter().map(|i| &self.nodes[i]).collect();
-        let children = children.into_iter().map(|i| &self.nodes[i]).collect();
-        (neighbors, children)
-    }
 }
 
 pub fn new_cluster_nodes<T: 'static>(
@@ -326,30 +245,15 @@ pub fn new_cluster_nodes<T: 'static>(
         .collect();
     let broadcast = TypeId::of::<T>() == TypeId::of::<BroadcastStage>();
     let stakes: Vec<u64> = nodes.iter().map(|node| node.stake).collect();
-    let mut weighted_shuffle = WeightedShuffle::new(&stakes).unwrap();
+    let mut weighted_shuffle = WeightedShuffle::new("cluster-nodes", &stakes);
     if broadcast {
         weighted_shuffle.remove_index(index[&self_pubkey]);
     }
-    // For backward compatibility:
-    //   * nodes which do not have contact-info are excluded.
-    //   * stakes are floored at 1.
-    // The sorting key here should be equivalent to
-    // solana_gossip::deprecated::sorted_stakes_with_index.
-    // Leader itself is excluded when sampling broadcast peers.
-    let compat_index = nodes
-        .iter()
-        .enumerate()
-        .filter(|(_, node)| node.contact_info().is_some())
-        .filter(|(_, node)| !broadcast || node.pubkey() != self_pubkey)
-        .sorted_by_key(|(_, node)| Reverse((node.stake.max(1), node.pubkey())))
-        .map(|(index, node)| (node.stake.max(1), index))
-        .collect();
     ClusterNodes {
         pubkey: self_pubkey,
         nodes,
         index,
         weighted_shuffle,
-        compat_index,
         _phantom: PhantomData::default(),
     }
 }
@@ -387,21 +291,6 @@ fn get_nodes(cluster_info: &ClusterInfo, stakes: &HashMap<Pubkey, u64>) -> Vec<N
     .collect()
 }
 
-fn enable_turbine_peers_shuffle_patch(shred_slot: Slot, root_bank: &Bank) -> bool {
-    let feature_slot = root_bank
-        .feature_set
-        .activated_slot(&feature_set::turbine_peers_shuffle::id());
-    match feature_slot {
-        None => false,
-        Some(feature_slot) => {
-            let epoch_schedule = root_bank.epoch_schedule();
-            let feature_epoch = epoch_schedule.get_epoch(feature_slot);
-            let shred_epoch = epoch_schedule.get_epoch(shred_slot);
-            feature_epoch < shred_epoch
-        }
-    }
-}
-
 impl<T> ClusterNodesCache<T> {
     pub fn new(
         // Capacity of underlying LRU-cache in terms of number of epochs.
@@ -528,42 +417,16 @@ pub fn make_test_cluster<R: Rng>(
 
 #[cfg(test)]
 mod tests {
-    use {
-        super::*,
-        solana_gossip::deprecated::{
-            shuffle_peers_and_index, sorted_retransmit_peers_and_stakes, sorted_stakes_with_index,
-        },
-    };
-
-    // Legacy methods copied for testing backward compatibility.
-
-    fn get_broadcast_peers(
-        cluster_info: &ClusterInfo,
-        stakes: Option<&HashMap<Pubkey, u64>>,
-    ) -> (Vec<ContactInfo>, Vec<(u64, usize)>) {
-        let mut peers = cluster_info.tvu_peers();
-        let peers_and_stakes = stake_weight_peers(&mut peers, stakes);
-        (peers, peers_and_stakes)
-    }
-
-    fn stake_weight_peers(
-        peers: &mut Vec<ContactInfo>,
-        stakes: Option<&HashMap<Pubkey, u64>>,
-    ) -> Vec<(u64, usize)> {
-        peers.dedup();
-        sorted_stakes_with_index(peers, stakes)
-    }
+    use super::*;
 
     #[test]
     fn test_cluster_nodes_retransmit() {
         let mut rng = rand::thread_rng();
         let (nodes, stakes, cluster_info) = make_test_cluster(&mut rng, 1_000, None);
-        let this_node = cluster_info.my_contact_info();
         // ClusterInfo::tvu_peers excludes the node itself.
         assert_eq!(cluster_info.tvu_peers().len(), nodes.len() - 1);
         let cluster_nodes = new_cluster_nodes::<RetransmitStage>(&cluster_info, &stakes);
         // All nodes with contact-info should be in the index.
-        assert_eq!(cluster_nodes.compat_index.len(), nodes.len());
         // Staked nodes with no contact-info should be included.
         assert!(cluster_nodes.nodes.len() > nodes.len());
         // Assert that all nodes keep their contact-info.
@@ -583,56 +446,6 @@ mod tests {
                 }
             }
         }
-        let (peers, stakes_and_index) =
-            sorted_retransmit_peers_and_stakes(&cluster_info, Some(&stakes));
-        assert_eq!(stakes_and_index.len(), peers.len());
-        assert_eq!(cluster_nodes.compat_index.len(), peers.len());
-        for (i, node) in cluster_nodes
-            .compat_index
-            .iter()
-            .map(|(_, i)| &cluster_nodes.nodes[*i])
-            .enumerate()
-        {
-            let (stake, index) = stakes_and_index[i];
-            // Wallclock may be update by ClusterInfo::push_self.
-            if node.pubkey() == this_node.id {
-                assert_eq!(this_node.id, peers[index].id)
-            } else {
-                assert_eq!(node.contact_info().unwrap(), &peers[index]);
-            }
-            assert_eq!(node.stake.max(1), stake);
-        }
-        let slot_leader = nodes[1..].choose(&mut rng).unwrap().id;
-        // Remove slot leader from peers indices.
-        let stakes_and_index: Vec<_> = stakes_and_index
-            .into_iter()
-            .filter(|(_stake, index)| peers[*index].id != slot_leader)
-            .collect();
-        assert_eq!(peers.len(), stakes_and_index.len() + 1);
-        let mut shred_seed = [0u8; 32];
-        rng.fill(&mut shred_seed[..]);
-        let (self_index, shuffled_peers_and_stakes) =
-            shuffle_peers_and_index(&this_node.id, &peers, &stakes_and_index, shred_seed);
-        let shuffled_index: Vec<_> = shuffled_peers_and_stakes
-            .into_iter()
-            .map(|(_, index)| index)
-            .collect();
-        assert_eq!(this_node.id, peers[shuffled_index[self_index]].id);
-        for fanout in 1..200 {
-            let (neighbors_indices, children_indices) =
-                compute_retransmit_peers(fanout, self_index, &shuffled_index);
-            let (neighbors, children) =
-                cluster_nodes.get_retransmit_peers_compat(shred_seed, fanout, slot_leader);
-            assert_eq!(children.len(), children_indices.len());
-            for (node, index) in children.into_iter().zip(children_indices) {
-                assert_eq!(*node.contact_info().unwrap(), peers[index]);
-            }
-            assert_eq!(neighbors.len(), neighbors_indices.len());
-            assert_eq!(neighbors[0].pubkey(), peers[neighbors_indices[0]].id);
-            for (node, index) in neighbors.into_iter().zip(neighbors_indices).skip(1) {
-                assert_eq!(*node.contact_info().unwrap(), peers[index]);
-            }
-        }
     }
 
     #[test]
@@ -644,7 +457,6 @@ mod tests {
         let cluster_nodes = ClusterNodes::<BroadcastStage>::new(&cluster_info, &stakes);
         // All nodes with contact-info should be in the index.
         // Excluding this node itself.
-        assert_eq!(cluster_nodes.compat_index.len() + 1, nodes.len());
         // Staked nodes with no contact-info should be included.
         assert!(cluster_nodes.nodes.len() > nodes.len());
         // Assert that all nodes keep their contact-info.
@@ -664,25 +476,5 @@ mod tests {
                 }
             }
         }
-        let (peers, peers_and_stakes) = get_broadcast_peers(&cluster_info, Some(&stakes));
-        assert_eq!(peers_and_stakes.len(), peers.len());
-        assert_eq!(cluster_nodes.compat_index.len(), peers.len());
-        for (i, node) in cluster_nodes
-            .compat_index
-            .iter()
-            .map(|(_, i)| &cluster_nodes.nodes[*i])
-            .enumerate()
-        {
-            let (stake, index) = peers_and_stakes[i];
-            assert_eq!(node.contact_info().unwrap(), &peers[index]);
-            assert_eq!(node.stake.max(1), stake);
-        }
-        for _ in 0..100 {
-            let mut shred_seed = [0u8; 32];
-            rng.fill(&mut shred_seed[..]);
-            let index = weighted_best(&peers_and_stakes, shred_seed);
-            let peer = cluster_nodes.get_broadcast_peer(shred_seed).unwrap();
-            assert_eq!(*peer, peers[index]);
-        }
     }
 }

core/src/consensus.rs

@@ -10,7 +10,7 @@ use {
     solana_ledger::{ancestor_iterator::AncestorIterator, blockstore::Blockstore, blockstore_db},
     solana_runtime::{
         bank::Bank, bank_forks::BankForks, commitment::VOTE_THRESHOLD_SIZE,
-        vote_account::VoteAccount,
+        vote_account::VoteAccountsHashMap,
     },
     solana_sdk::{
         clock::{Slot, UnixTimestamp},
@@ -253,7 +253,7 @@ impl Tower {
     pub(crate) fn collect_vote_lockouts(
         vote_account_pubkey: &Pubkey,
         bank_slot: Slot,
-        vote_accounts: &HashMap<Pubkey, (/*stake:*/ u64, VoteAccount)>,
+        vote_accounts: &VoteAccountsHashMap,
         ancestors: &HashMap<Slot, HashSet<Slot>>,
         get_frozen_hash: impl Fn(Slot) -> Option<Hash>,
         latest_validator_votes_for_frozen_banks: &mut LatestValidatorVotesForFrozenBanks,
@@ -636,7 +636,7 @@ impl Tower {
         descendants: &HashMap<Slot, HashSet<u64>>,
         progress: &ProgressMap,
         total_stake: u64,
-        epoch_vote_accounts: &HashMap<Pubkey, (u64, VoteAccount)>,
+        epoch_vote_accounts: &VoteAccountsHashMap,
         latest_validator_votes_for_frozen_banks: &LatestValidatorVotesForFrozenBanks,
         heaviest_subtree_fork_choice: &HeaviestSubtreeForkChoice,
     ) -> SwitchForkDecision {
@@ -929,7 +929,7 @@ impl Tower {
         descendants: &HashMap<Slot, HashSet<u64>>,
         progress: &ProgressMap,
         total_stake: u64,
-        epoch_vote_accounts: &HashMap<Pubkey, (u64, VoteAccount)>,
+        epoch_vote_accounts: &VoteAccountsHashMap,
         latest_validator_votes_for_frozen_banks: &LatestValidatorVotesForFrozenBanks,
         heaviest_subtree_fork_choice: &HeaviestSubtreeForkChoice,
     ) -> SwitchForkDecision {
@@ -1377,7 +1377,7 @@ pub mod test {
         },
         itertools::Itertools,
         solana_ledger::{blockstore::make_slot_entries, get_tmp_ledger_path},
-        solana_runtime::bank::Bank,
+        solana_runtime::{bank::Bank, vote_account::VoteAccount},
         solana_sdk::{
             account::{Account, AccountSharedData, ReadableAccount, WritableAccount},
             clock::Slot,
@@ -1398,7 +1398,7 @@ pub mod test {
         trees::tr,
     };
 
-    fn gen_stakes(stake_votes: &[(u64, &[u64])]) -> HashMap<Pubkey, (u64, VoteAccount)> {
+    fn gen_stakes(stake_votes: &[(u64, &[u64])]) -> VoteAccountsHashMap {
         stake_votes
             .iter()
             .map(|(lamports, votes)| {

core/src/cost_update_service.rs

@@ -134,10 +134,6 @@ impl CostUpdateService {
                 .upsert_instruction_cost(program_id, units);
             update_count += 1;
         }
-        debug!(
-           "after replayed into bank, updated cost model instruction cost table, current values: {:?}",
-           cost_model.read().unwrap().get_instruction_cost_table()
-        );
         update_count
     }
 }
@@ -150,15 +146,10 @@ mod tests {
     fn test_update_cost_model_with_empty_execute_timings() {
         let cost_model = Arc::new(RwLock::new(CostModel::default()));
         let mut empty_execute_timings = ExecuteTimings::default();
-        CostUpdateService::update_cost_model(&cost_model, &mut empty_execute_timings);
 
         assert_eq!(
             0,
-            cost_model
-                .read()
-                .unwrap()
-                .get_instruction_cost_table()
-                .len()
+            CostUpdateService::update_cost_model(&cost_model, &mut empty_execute_timings),
         );
     }
 
@@ -188,22 +179,16 @@ mod tests {
                     total_errored_units,
                 },
             );
-            CostUpdateService::update_cost_model(&cost_model, &mut execute_timings);
             assert_eq!(
                 1,
-                cost_model
-                    .read()
-                    .unwrap()
-                    .get_instruction_cost_table()
-                    .len()
+                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings),
             );
             assert_eq!(
-                Some(&expected_cost),
+                expected_cost,
                 cost_model
                     .read()
                     .unwrap()
-                    .get_instruction_cost_table()
-                    .get(&program_key_1)
+                    .find_instruction_cost(&program_key_1)
             );
         }
 
@@ -225,22 +210,16 @@ mod tests {
                     total_errored_units: 0,
                 },
             );
-            CostUpdateService::update_cost_model(&cost_model, &mut execute_timings);
             assert_eq!(
                 1,
-                cost_model
-                    .read()
-                    .unwrap()
-                    .get_instruction_cost_table()
-                    .len()
+                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings),
             );
             assert_eq!(
-                Some(&expected_cost),
+                expected_cost,
                 cost_model
                     .read()
                     .unwrap()
-                    .get_instruction_cost_table()
-                    .get(&program_key_1)
+                    .find_instruction_cost(&program_key_1)
             );
         }
     }
@@ -264,20 +243,46 @@ mod tests {
                     total_errored_units: 0,
                 },
             );
-            CostUpdateService::update_cost_model(&cost_model, &mut execute_timings);
             // If both the `errored_txs_compute_consumed` is empty and `count == 0`, then
             // nothing should be inserted into the cost model
-            assert!(cost_model
+            assert_eq!(
+                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings),
+                0
+            );
+        }
+
+        // set up current instruction cost to 100
+        let current_program_cost = 100;
+        {
+            execute_timings.details.per_program_timings.insert(
+                program_key_1,
+                ProgramTiming {
+                    accumulated_us: 1000,
+                    accumulated_units: current_program_cost,
+                    count: 1,
+                    errored_txs_compute_consumed: vec![],
+                    total_errored_units: 0,
+                },
+            );
+            assert_eq!(
+                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings),
+                1
+            );
+            assert_eq!(
+                current_program_cost,
+                cost_model
                     .read()
                     .unwrap()
-                .get_instruction_cost_table()
-                .is_empty());
+                    .find_instruction_cost(&program_key_1)
+            );
         }
 
         // Test updating cost model with only erroring compute costs where the `cost_per_error` is
         // greater than the current instruction cost for the program. Should update with the
         // new erroring compute costs
         let cost_per_error = 1000;
+        // the expect cost is (previous_cost + new_cost)/2 = (100 + 1000)/2 = 550
+        let expected_units = 550;
         {
             let errored_txs_compute_consumed = vec![cost_per_error; 3];
             let total_errored_units = errored_txs_compute_consumed.iter().sum();
@@ -291,29 +296,23 @@ mod tests {
                     total_errored_units,
                 },
             );
-            CostUpdateService::update_cost_model(&cost_model, &mut execute_timings);
             assert_eq!(
-                1,
-                cost_model
-                    .read()
-                    .unwrap()
-                    .get_instruction_cost_table()
-                    .len()
+                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings),
+                1
             );
             assert_eq!(
-                Some(&cost_per_error),
+                expected_units,
                 cost_model
                     .read()
                     .unwrap()
-                    .get_instruction_cost_table()
-                    .get(&program_key_1)
+                    .find_instruction_cost(&program_key_1)
             );
         }
 
         // Test updating cost model with only erroring compute costs where the error cost is
         // `smaller_cost_per_error`, less than the current instruction cost for the program.
         // The cost should not decrease for these new lesser errors
-        let smaller_cost_per_error = cost_per_error - 10;
+        let smaller_cost_per_error = expected_units - 10;
         {
             let errored_txs_compute_consumed = vec![smaller_cost_per_error; 3];
             let total_errored_units = errored_txs_compute_consumed.iter().sum();
@@ -327,22 +326,16 @@ mod tests {
                     total_errored_units,
                 },
             );
-            CostUpdateService::update_cost_model(&cost_model, &mut execute_timings);
             assert_eq!(
-                1,
-                cost_model
-                    .read()
-                    .unwrap()
-                    .get_instruction_cost_table()
-                    .len()
+                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings),
+                1
             );
             assert_eq!(
-                Some(&cost_per_error),
+                expected_units,
                 cost_model
                     .read()
                     .unwrap()
-                    .get_instruction_cost_table()
-                    .get(&program_key_1)
+                    .find_instruction_cost(&program_key_1)
             );
         }
     }

core/src/find_packet_sender_stake_stage.rs

@@ -0,0 +1,185 @@
+use {
+    crossbeam_channel::{Receiver, RecvTimeoutError, Sender},
+    rayon::{prelude::*, ThreadPool},
+    solana_gossip::cluster_info::ClusterInfo,
+    solana_measure::measure::Measure,
+    solana_perf::packet::PacketBatch,
+    solana_rayon_threadlimit::get_thread_count,
+    solana_runtime::bank_forks::BankForks,
+    solana_sdk::timing::timestamp,
+    solana_streamer::streamer::{self, StreamerError},
+    std::{
+        cell::RefCell,
+        collections::HashMap,
+        net::IpAddr,
+        sync::{Arc, RwLock},
+        thread::{self, Builder, JoinHandle},
+        time::{Duration, Instant},
+    },
+};
+
+const IP_TO_STAKE_REFRESH_DURATION: Duration = Duration::from_secs(5);
+
+thread_local!(static PAR_THREAD_POOL: RefCell<ThreadPool> = RefCell::new(rayon::ThreadPoolBuilder::new()
+                    .num_threads(get_thread_count())
+                    .thread_name(|ix| format!("transaction_sender_stake_stage_{}", ix))
+                    .build()
+                    .unwrap()));
+
+pub type FindPacketSenderStakeSender = Sender<Vec<PacketBatch>>;
+pub type FindPacketSenderStakeReceiver = Receiver<Vec<PacketBatch>>;
+
+#[derive(Debug, Default)]
+struct FindPacketSenderStakeStats {
+    last_print: u64,
+    refresh_ip_to_stake_time: u64,
+    apply_sender_stakes_time: u64,
+    send_batches_time: u64,
+    receive_batches_time: u64,
+    total_batches: u64,
+    total_packets: u64,
+}
+
+impl FindPacketSenderStakeStats {
+    fn report(&mut self) {
+        let now = timestamp();
+        let elapsed_ms = now - self.last_print;
+        if elapsed_ms > 2000 {
+            datapoint_info!(
+                "find_packet_sender_stake-services_stats",
+                (
+                    "refresh_ip_to_stake_time",
+                    self.refresh_ip_to_stake_time as i64,
+                    i64
+                ),
+                (
+                    "apply_sender_stakes_time",
+                    self.apply_sender_stakes_time as i64,
+                    i64
+                ),
+                ("send_batches_time", self.send_batches_time as i64, i64),
+                (
+                    "receive_batches_time",
+                    self.receive_batches_time as i64,
+                    i64
+                ),
+                ("total_batches", self.total_batches as i64, i64),
+                ("total_packets", self.total_packets as i64, i64),
+            );
+            *self = FindPacketSenderStakeStats::default();
+            self.last_print = now;
+        }
+    }
+}
+
+pub struct FindPacketSenderStakeStage {
+    thread_hdl: JoinHandle<()>,
+}
+
+impl FindPacketSenderStakeStage {
+    pub fn new(
+        packet_receiver: streamer::PacketBatchReceiver,
+        sender: FindPacketSenderStakeSender,
+        bank_forks: Arc<RwLock<BankForks>>,
+        cluster_info: Arc<ClusterInfo>,
+    ) -> Self {
+        let mut stats = FindPacketSenderStakeStats::default();
+        let thread_hdl = Builder::new()
+            .name("find-packet-sender-stake".to_string())
+            .spawn(move || {
+                let mut last_stakes = Instant::now();
+                let mut ip_to_stake: HashMap<IpAddr, u64> = HashMap::new();
+                loop {
+                    let mut refresh_ip_to_stake_time = Measure::start("refresh_ip_to_stake_time");
+                    Self::try_refresh_ip_to_stake(
+                        &mut last_stakes,
+                        &mut ip_to_stake,
+                        bank_forks.clone(),
+                        cluster_info.clone(),
+                    );
+                    refresh_ip_to_stake_time.stop();
+                    stats.refresh_ip_to_stake_time = stats
+                        .refresh_ip_to_stake_time
+                        .saturating_add(refresh_ip_to_stake_time.as_us());
+
+                    match streamer::recv_packet_batches(&packet_receiver) {
+                        Ok((mut batches, num_packets, recv_duration)) => {
+                            let num_batches = batches.len();
+                            let mut apply_sender_stakes_time =
+                                Measure::start("apply_sender_stakes_time");
+                            Self::apply_sender_stakes(&mut batches, &ip_to_stake);
+                            apply_sender_stakes_time.stop();
+
+                            let mut send_batches_time = Measure::start("send_batches_time");
+                            if let Err(e) = sender.send(batches) {
+                                info!("Sender error: {:?}", e);
+                            }
+                            send_batches_time.stop();
+
+                            stats.apply_sender_stakes_time = stats
+                                .apply_sender_stakes_time
+                                .saturating_add(apply_sender_stakes_time.as_us());
+                            stats.send_batches_time = stats
+                                .send_batches_time
+                                .saturating_add(send_batches_time.as_us());
+                            stats.receive_batches_time = stats
+                                .receive_batches_time
+                                .saturating_add(recv_duration.as_nanos() as u64);
+                            stats.total_batches =
+                                stats.total_batches.saturating_add(num_batches as u64);
+                            stats.total_packets =
+                                stats.total_packets.saturating_add(num_packets as u64);
+                        }
+                        Err(e) => match e {
+                            StreamerError::RecvTimeout(RecvTimeoutError::Disconnected) => break,
+                            StreamerError::RecvTimeout(RecvTimeoutError::Timeout) => (),
+                            _ => error!("error: {:?}", e),
+                        },
+                    }
+
+                    stats.report();
+                }
+            })
+            .unwrap();
+        Self { thread_hdl }
+    }
+
+    fn try_refresh_ip_to_stake(
+        last_stakes: &mut Instant,
+        ip_to_stake: &mut HashMap<IpAddr, u64>,
+        bank_forks: Arc<RwLock<BankForks>>,
+        cluster_info: Arc<ClusterInfo>,
+    ) {
+        if last_stakes.elapsed() > IP_TO_STAKE_REFRESH_DURATION {
+            let root_bank = bank_forks.read().unwrap().root_bank();
+            let staked_nodes = root_bank.staked_nodes();
+            *ip_to_stake = cluster_info
+                .tvu_peers()
+                .into_iter()
+                .filter_map(|node| {
+                    let stake = staked_nodes.get(&node.id)?;
+                    Some((node.tvu.ip(), *stake))
+                })
+                .collect();
+            *last_stakes = Instant::now();
+        }
+    }
+
+    fn apply_sender_stakes(batches: &mut [PacketBatch], ip_to_stake: &HashMap<IpAddr, u64>) {
+        PAR_THREAD_POOL.with(|thread_pool| {
+            thread_pool.borrow().install(|| {
+                batches
+                    .into_par_iter()
+                    .flat_map(|batch| batch.packets.par_iter_mut())
+                    .for_each(|packet| {
+                        packet.meta.sender_stake =
+                            *ip_to_stake.get(&packet.meta.addr().ip()).unwrap_or(&0);
+                    });
+            })
+        });
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.thread_hdl.join()
+    }
+}

core/src/leader_slot_banking_stage_timing_metrics.rs

@@ -24,7 +24,6 @@ impl LeaderExecuteAndCommitTimings {
         saturating_add_assign!(self.record_us, other.record_us);
         saturating_add_assign!(self.commit_us, other.commit_us);
         saturating_add_assign!(self.find_and_send_votes_us, other.find_and_send_votes_us);
-        saturating_add_assign!(self.commit_us, other.commit_us);
         self.record_transactions_timings
             .accumulate(&other.record_transactions_timings);
         self.execute_timings.accumulate(&other.execute_timings);

core/src/ledger_cleanup_service.rs

@@ -1,4 +1,8 @@
-//! The `ledger_cleanup_service` drops older ledger data to limit disk space usage
+//! The `ledger_cleanup_service` drops older ledger data to limit disk space usage.
+//! The service works by counting the number of live data shreds in the ledger; this
+//! can be done quickly and should have a fairly stable correlation to actual bytes.
+//! Once the shred count (and thus roughly the byte count) reaches a threshold,
+//! the services begins removing data in FIFO order.
 
 use {
     crossbeam_channel::{Receiver, RecvTimeoutError},

core/src/ledger_metric_report_service.rs

@@ -13,7 +13,9 @@ use {
     },
 };
 
-// Determines how often we report blockstore metrics.
+// Determines how often we report blockstore metrics under
+// LedgerMetricReportService.  Note that there're other blockstore
+// metrics that are reported outside LedgerMetricReportService.
 const BLOCKSTORE_METRICS_REPORT_PERIOD_MILLIS: u64 = 10000;
 
 pub struct LedgerMetricReportService {

core/src/lib.rs

@@ -24,6 +24,7 @@ pub mod cost_update_service;
 pub mod drop_bank_service;
 pub mod duplicate_repair_status;
 pub mod fetch_stage;
+pub mod find_packet_sender_stake_stage;
 pub mod fork_choice;
 pub mod gen_keys;
 pub mod heaviest_subtree_fork_choice;
@@ -36,6 +37,8 @@ pub mod optimistic_confirmation_verifier;
 pub mod outstanding_requests;
 pub mod packet_hasher;
 pub mod packet_threshold;
+pub mod poh_timing_report_service;
+pub mod poh_timing_reporter;
 pub mod progress_map;
 pub mod qos_service;
 pub mod repair_generic_traversal;

core/src/poh_timing_report_service.rs

@@ -0,0 +1,87 @@
+//! PohTimingReportService module
+use {
+    crate::poh_timing_reporter::PohTimingReporter,
+    solana_metrics::poh_timing_point::{PohTimingReceiver, SlotPohTimingInfo},
+    std::{
+        string::ToString,
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc,
+        },
+        thread::{self, Builder, JoinHandle},
+        time::Duration,
+    },
+};
+
+/// Timeout to wait on the poh timing points from the channel
+const POH_TIMING_RECEIVER_TIMEOUT_MILLISECONDS: u64 = 1000;
+
+/// The `poh_timing_report_service` receives signals of relevant timing points
+/// during the processing of a slot, (i.e. from blockstore and poh), aggregate and
+/// report the result as datapoints.
+pub struct PohTimingReportService {
+    t_poh_timing: JoinHandle<()>,
+}
+
+impl PohTimingReportService {
+    pub fn new(receiver: PohTimingReceiver, exit: Arc<AtomicBool>) -> Self {
+        let exit_signal = exit;
+        let mut poh_timing_reporter = PohTimingReporter::default();
+        let t_poh_timing = Builder::new()
+            .name("poh_timing_report".to_string())
+            .spawn(move || loop {
+                if exit_signal.load(Ordering::Relaxed) {
+                    break;
+                }
+                if let Ok(SlotPohTimingInfo {
+                    slot,
+                    root_slot,
+                    timing_point,
+                }) = receiver.recv_timeout(Duration::from_millis(
+                    POH_TIMING_RECEIVER_TIMEOUT_MILLISECONDS,
+                )) {
+                    poh_timing_reporter.process(slot, root_slot, timing_point);
+                }
+            })
+            .unwrap();
+        Self { t_poh_timing }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.t_poh_timing.join()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use {
+        super::*, crossbeam_channel::unbounded, solana_metrics::poh_timing_point::SlotPohTimingInfo,
+    };
+
+    #[test]
+    /// Test the life cycle of the PohTimingReportService
+    fn test_poh_timing_report_service() {
+        let (poh_timing_point_sender, poh_timing_point_receiver) = unbounded();
+        let exit = Arc::new(AtomicBool::new(false));
+        // Create the service
+        let poh_timing_report_service =
+            PohTimingReportService::new(poh_timing_point_receiver, exit.clone());
+
+        // Send SlotPohTimingPoint
+        let _ = poh_timing_point_sender.send(SlotPohTimingInfo::new_slot_start_poh_time_point(
+            42, None, 100,
+        ));
+        let _ = poh_timing_point_sender.send(SlotPohTimingInfo::new_slot_end_poh_time_point(
+            42, None, 200,
+        ));
+        let _ = poh_timing_point_sender.send(SlotPohTimingInfo::new_slot_full_poh_time_point(
+            42, None, 150,
+        ));
+
+        // Shutdown the service
+        exit.store(true, Ordering::Relaxed);
+        poh_timing_report_service
+            .join()
+            .expect("poh_timing_report_service completed");
+    }
+}

core/src/poh_timing_reporter.rs

@@ -0,0 +1,239 @@
+//! A poh_timing_reporter module implement poh timing point and timing reporter
+//! structs.
+use {
+    solana_metrics::{datapoint_info, poh_timing_point::PohTimingPoint},
+    solana_sdk::clock::Slot,
+    std::{collections::HashMap, fmt},
+};
+
+/// A SlotPohTimestamp records timing of the events during the processing of a
+/// slot by the validator
+#[derive(Debug, Clone, Copy, Default)]
+pub struct SlotPohTimestamp {
+    /// Slot start time from poh
+    pub start_time: u64,
+    /// Slot end time from poh
+    pub end_time: u64,
+    /// Last shred received time from block producer
+    pub full_time: u64,
+}
+
+/// Display trait
+impl fmt::Display for SlotPohTimestamp {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "SlotPohTimestamp: start={} end={} full={}",
+            self.start_time, self.end_time, self.full_time
+        )
+    }
+}
+
+impl SlotPohTimestamp {
+    /// Return true if the timing points of all events are received.
+    pub fn is_complete(&self) -> bool {
+        self.start_time != 0 && self.end_time != 0 && self.full_time != 0
+    }
+
+    /// Update with timing point
+    pub fn update(&mut self, timing_point: PohTimingPoint) {
+        match timing_point {
+            PohTimingPoint::PohSlotStart(ts) => self.start_time = ts,
+            PohTimingPoint::PohSlotEnd(ts) => self.end_time = ts,
+            PohTimingPoint::FullSlotReceived(ts) => self.full_time = ts,
+        }
+    }
+
+    /// Return the time difference from slot start to slot full
+    fn slot_start_to_full_time(&self) -> i64 {
+        (self.full_time as i64).saturating_sub(self.start_time as i64)
+    }
+
+    /// Return the time difference from slot full to slot end
+    fn slot_full_to_end_time(&self) -> i64 {
+        (self.end_time as i64).saturating_sub(self.full_time as i64)
+    }
+
+    /// Report PohTiming for a slot
+    pub fn report(&self, slot: Slot) {
+        datapoint_info!(
+            "poh_slot_timing",
+            ("slot", slot as i64, i64),
+            ("start_time", self.start_time as i64, i64),
+            ("end_time", self.end_time as i64, i64),
+            ("full_time", self.full_time as i64, i64),
+            (
+                "start_to_full_time_diff",
+                self.slot_start_to_full_time(),
+                i64
+            ),
+            ("full_to_end_time_diff", self.slot_full_to_end_time(), i64),
+        );
+    }
+}
+
+/// A PohTimingReporter manages and reports the timing of events for incoming
+/// slots
+#[derive(Default)]
+pub struct PohTimingReporter {
+    /// Storage map of SlotPohTimestamp per slot
+    slot_timestamps: HashMap<Slot, SlotPohTimestamp>,
+    last_root_slot: Slot,
+}
+
+impl PohTimingReporter {
+    /// Return true if PohTiming is complete for the slot
+    pub fn is_complete(&self, slot: Slot) -> bool {
+        if let Some(slot_timestamp) = self.slot_timestamps.get(&slot) {
+            slot_timestamp.is_complete()
+        } else {
+            false
+        }
+    }
+
+    /// Process incoming PohTimingPoint from the channel
+    pub fn process(&mut self, slot: Slot, root_slot: Option<Slot>, t: PohTimingPoint) -> bool {
+        let slot_timestamp = self
+            .slot_timestamps
+            .entry(slot)
+            .or_insert_with(SlotPohTimestamp::default);
+
+        slot_timestamp.update(t);
+        let is_completed = slot_timestamp.is_complete();
+        if is_completed {
+            slot_timestamp.report(slot);
+        }
+
+        // delete slots that are older than the root_slot
+        if let Some(root_slot) = root_slot {
+            if root_slot > self.last_root_slot {
+                self.slot_timestamps.retain(|&k, _| k >= root_slot);
+                self.last_root_slot = root_slot;
+            }
+        }
+        is_completed
+    }
+
+    /// Return the count of slot_timestamps in tracking
+    pub fn slot_count(&self) -> usize {
+        self.slot_timestamps.len()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    /// Test poh_timing_reporter
+    fn test_poh_timing_reporter() {
+        // create a reporter
+        let mut reporter = PohTimingReporter::default();
+
+        // process all relevant PohTimingPoints for slot 42
+        let complete = reporter.process(42, None, PohTimingPoint::PohSlotStart(100));
+        assert!(!complete);
+        let complete = reporter.process(42, None, PohTimingPoint::PohSlotEnd(200));
+        assert!(!complete);
+        let complete = reporter.process(42, None, PohTimingPoint::FullSlotReceived(150));
+        // assert that the PohTiming is complete
+        assert!(complete);
+
+        // Move root to slot 43
+        let root = Some(43);
+
+        // process all relevant PohTimingPoints for slot 45
+        let complete = reporter.process(45, None, PohTimingPoint::PohSlotStart(100));
+        assert!(!complete);
+        let complete = reporter.process(45, None, PohTimingPoint::PohSlotEnd(200));
+        assert!(!complete);
+        let complete = reporter.process(45, root, PohTimingPoint::FullSlotReceived(150));
+        // assert that the PohTiming is complete
+        assert!(complete);
+
+        // assert that only one timestamp remains in track
+        assert_eq!(reporter.slot_count(), 1)
+    }
+
+    #[test]
+    /// Test poh_timing_reporter
+    fn test_poh_timing_reporter_out_of_order() {
+        // create a reporter
+        let mut reporter = PohTimingReporter::default();
+
+        // process all relevant PohTimingPoints for slot 42/43 out of order
+        let mut c = 0;
+        // slot_start 42
+        c += reporter.process(42, None, PohTimingPoint::PohSlotStart(100)) as i32;
+        // slot_full 42
+        c += reporter.process(42, None, PohTimingPoint::FullSlotReceived(120)) as i32;
+        // slot_full 43
+        c += reporter.process(43, None, PohTimingPoint::FullSlotReceived(140)) as i32;
+        // slot_end 42
+        c += reporter.process(42, None, PohTimingPoint::PohSlotEnd(200)) as i32;
+        // slot start 43
+        c += reporter.process(43, None, PohTimingPoint::PohSlotStart(100)) as i32;
+        // slot end 43
+        c += reporter.process(43, None, PohTimingPoint::PohSlotEnd(200)) as i32;
+
+        // assert that both timing points are complete
+        assert_eq!(c, 2);
+
+        // assert that both timestamps remain in track
+        assert_eq!(reporter.slot_count(), 2)
+    }
+
+    #[test]
+    /// Test poh_timing_reporter
+    fn test_poh_timing_reporter_never_complete() {
+        // create a reporter
+        let mut reporter = PohTimingReporter::default();
+
+        let mut c = 0;
+
+        // process all relevant PohTimingPoints for slot 42/43 out of order
+        // slot_start 42
+        c += reporter.process(42, None, PohTimingPoint::PohSlotStart(100)) as i32;
+
+        // slot_full 42
+        c += reporter.process(42, None, PohTimingPoint::FullSlotReceived(120)) as i32;
+
+        // slot_full 43
+        c += reporter.process(43, None, PohTimingPoint::FullSlotReceived(140)) as i32;
+
+        // skip slot 42, jump to slot 43
+        // slot start 43
+        c += reporter.process(43, None, PohTimingPoint::PohSlotStart(100)) as i32;
+
+        // slot end 43
+        c += reporter.process(43, None, PohTimingPoint::PohSlotEnd(200)) as i32;
+
+        // assert that only one timing point is complete
+        assert_eq!(c, 1);
+
+        // assert that both timestamp is in track
+        assert_eq!(reporter.slot_count(), 2)
+    }
+
+    #[test]
+    fn test_poh_timing_reporter_overflow() {
+        // create a reporter
+        let mut reporter = PohTimingReporter::default();
+
+        // process all relevant PohTimingPoints for a slot
+        let complete = reporter.process(42, None, PohTimingPoint::PohSlotStart(1647624609896));
+        assert!(!complete);
+        let complete = reporter.process(42, None, PohTimingPoint::PohSlotEnd(1647624610286));
+        assert!(!complete);
+        let complete = reporter.process(42, None, PohTimingPoint::FullSlotReceived(1647624610281));
+
+        // assert that the PohTiming is complete
+        assert!(complete);
+    }
+
+    #[test]
+    fn test_slot_poh_timestamp_fmt() {
+        let t = SlotPohTimestamp::default();
+        assert_eq!(format!("{}", t), "SlotPohTimestamp: start=0 end=0 full=0");
+    }
+}

core/src/progress_map.rs

@@ -7,7 +7,7 @@ use {
     },
     solana_ledger::blockstore_processor::{ConfirmationProgress, ConfirmationTiming},
     solana_program_runtime::timings::ExecuteTimingType,
-    solana_runtime::{bank::Bank, bank_forks::BankForks, vote_account::VoteAccount},
+    solana_runtime::{bank::Bank, bank_forks::BankForks, vote_account::VoteAccountsHashMap},
     solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey},
     std::{
         collections::{BTreeMap, HashMap, HashSet},
@@ -516,7 +516,7 @@ impl PropagatedStats {
         &mut self,
         node_pubkey: &Pubkey,
         vote_account_pubkeys: &[Pubkey],
-        epoch_vote_accounts: &HashMap<Pubkey, (u64, VoteAccount)>,
+        epoch_vote_accounts: &VoteAccountsHashMap,
     ) {
         self.propagated_node_ids.insert(*node_pubkey);
         for vote_account_pubkey in vote_account_pubkeys.iter() {
@@ -695,7 +695,7 @@ impl ProgressMap {
 
 #[cfg(test)]
 mod test {
-    use super::*;
+    use {super::*, solana_runtime::vote_account::VoteAccount};
 
     #[test]
     fn test_add_vote_pubkey() {

core/src/qos_service.rs

@@ -133,7 +133,7 @@ impl QosService {
         let mut num_included = 0;
         let select_results = transactions
             .zip(transactions_costs)
-            .map(|(tx, cost)| match cost_tracker.try_add(tx, cost) {
+            .map(|(tx, cost)| match cost_tracker.try_add(cost) {
                 Ok(current_block_cost) => {
                     debug!("slot {:?}, transaction {:?}, cost {:?}, fit into current block, current block cost {}", bank.slot(), tx, cost, current_block_cost);
                     self.metrics.stats.selected_txs_count.fetch_add(1, Ordering::Relaxed);
@@ -170,6 +170,35 @@ impl QosService {
         (select_results, num_included)
     }
 
+    /// Update the transaction cost in the cost_tracker with the real cost for
+    /// transactions that were executed successfully;
+    /// Otherwise remove the cost from the cost tracker, therefore preventing cost_tracker
+    /// being inflated with unsuccessfully executed transactions.
+    pub fn update_or_remove_transaction_costs<'a>(
+        transaction_costs: impl Iterator<Item = &'a TransactionCost>,
+        transaction_qos_results: impl Iterator<Item = &'a transaction::Result<()>>,
+        retryable_transaction_indexes: &[usize],
+        bank: &Arc<Bank>,
+    ) {
+        let mut cost_tracker = bank.write_cost_tracker().unwrap();
+        transaction_costs
+            .zip(transaction_qos_results)
+            .enumerate()
+            .for_each(|(index, (tx_cost, qos_inclusion_result))| {
+                // Only transactions that the qos service incuded have been added to the
+                // cost tracker.
+                if qos_inclusion_result.is_ok() && retryable_transaction_indexes.contains(&index) {
+                    cost_tracker.remove(tx_cost);
+                } else {
+                    // TODO: Update the cost tracker with the actual execution compute units.
+                    // Will have to plumb it in next; For now, keep estimated costs.
+                    //
+                    // let actual_execution_cost = 0;
+                    // cost_tracker.update_execution_cost(tx_cost, actual_execution_cost);
+                }
+            });
+    }
+
     // metrics are reported by bank slot
     pub fn report_metrics(&self, bank: Arc<Bank>) {
         self.report_sender

core/src/replay_stage.rs

@@ -1682,6 +1682,9 @@ impl ReplayStage {
         blockstore
             .set_dead_slot(slot)
             .expect("Failed to mark slot as dead in blockstore");
+
+        blockstore.slots_stats.mark_dead(slot);
+
         rpc_subscriptions.notify_slot_update(SlotUpdate::Dead {
             slot,
             err: format!("error: {:?}", err),
@@ -1788,6 +1791,9 @@ impl ReplayStage {
                 epoch_slots_frozen_slots,
                 drop_bank_sender,
             );
+
+            blockstore.slots_stats.mark_rooted(new_root);
+
             rpc_subscriptions.notify_roots(rooted_slots);
             if let Some(sender) = bank_notification_sender {
                 sender
@@ -2301,7 +2307,7 @@ impl ReplayStage {
             }
         }
 
-        // send accumulated excute-timings to cost_update_service
+        // send accumulated execute-timings to cost_update_service
         if !execute_timings.details.per_program_timings.is_empty() {
             cost_update_sender
                 .send(CostUpdate::ExecuteTiming {
@@ -2589,7 +2595,7 @@ impl ReplayStage {
                     */
 
                     // Imagine 90% of validators voted on slot 4, but only 9% landed. If everybody that fails
-                    // the switch theshold abandons slot 4 to build on slot 8 (because it's *currently* heavier),
+                    // the switch threshold abandons slot 4 to build on slot 8 (because it's *currently* heavier),
                     // then there will be no blocks to include the votes for slot 4, and the network halts
                     // because 90% of validators can't vote
                     info!(
@@ -2931,6 +2937,7 @@ impl ReplayStage {
             accounts_background_request_sender,
             highest_confirmed_root,
         );
+
         drop_bank_sender
             .send(removed_banks)
             .unwrap_or_else(|err| warn!("bank drop failed: {:?}", err));

core/src/retransmit_stage.rs

@@ -240,7 +240,7 @@ fn retransmit(
     epoch_fetch.stop();
     stats.epoch_fetch += epoch_fetch.as_us();
 
-    let mut epoch_cache_update = Measure::start("retransmit_epoch_cach_update");
+    let mut epoch_cache_update = Measure::start("retransmit_epoch_cache_update");
     maybe_reset_shreds_received_cache(shreds_received, hasher_reset_ts);
     epoch_cache_update.stop();
     stats.epoch_cache_update += epoch_cache_update.as_us();
@@ -525,85 +525,7 @@ impl RetransmitStage {
 
 #[cfg(test)]
 mod tests {
-    use {
-        super::*,
-        solana_gossip::contact_info::ContactInfo,
-        solana_ledger::{
-            blockstore_processor::{test_process_blockstore, ProcessOptions},
-            create_new_tmp_ledger,
-            genesis_utils::{create_genesis_config, GenesisConfigInfo},
-        },
-        solana_net_utils::find_available_port_in_range,
-        solana_sdk::signature::Keypair,
-        solana_streamer::socket::SocketAddrSpace,
-        std::net::{IpAddr, Ipv4Addr},
-    };
-
-    #[test]
-    fn test_skip_repair() {
-        solana_logger::setup();
-        let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(123);
-        let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config);
-        let blockstore = Blockstore::open(&ledger_path).unwrap();
-        let opts = ProcessOptions {
-            accounts_db_test_hash_calculation: true,
-            full_leader_cache: true,
-            ..ProcessOptions::default()
-        };
-        let (bank_forks, leader_schedule_cache) =
-            test_process_blockstore(&genesis_config, &blockstore, opts);
-        let leader_schedule_cache = Arc::new(leader_schedule_cache);
-        let bank_forks = Arc::new(RwLock::new(bank_forks));
-
-        let mut me = ContactInfo::new_localhost(&solana_sdk::pubkey::new_rand(), 0);
-        let ip_addr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0));
-        let port = find_available_port_in_range(ip_addr, (8000, 10000)).unwrap();
-        let me_retransmit = UdpSocket::bind(format!("127.0.0.1:{}", port)).unwrap();
-        // need to make sure tvu and tpu are valid addresses
-        me.tvu_forwards = me_retransmit.local_addr().unwrap();
-
-        let port = find_available_port_in_range(ip_addr, (8000, 10000)).unwrap();
-        me.tvu = UdpSocket::bind(format!("127.0.0.1:{}", port))
-            .unwrap()
-            .local_addr()
-            .unwrap();
-        // This fixes the order of nodes returned by shuffle_peers_and_index,
-        // and makes turbine retransmit tree deterministic for the purpose of
-        // the test.
-        let other = std::iter::repeat_with(solana_sdk::pubkey::new_rand)
-            .find(|pk| me.id < *pk)
-            .unwrap();
-        let other = ContactInfo::new_localhost(&other, 0);
-        let cluster_info = ClusterInfo::new(
-            other,
-            Arc::new(Keypair::new()),
-            SocketAddrSpace::Unspecified,
-        );
-        cluster_info.insert_info(me);
-
-        let retransmit_socket = Arc::new(vec![UdpSocket::bind("0.0.0.0:0").unwrap()]);
-        let cluster_info = Arc::new(cluster_info);
-
-        let (retransmit_sender, retransmit_receiver) = unbounded();
-        let _retransmit_sender = retransmit_sender.clone();
-        let _t_retransmit = retransmitter(
-            retransmit_socket,
-            bank_forks,
-            leader_schedule_cache,
-            cluster_info,
-            retransmit_receiver,
-            Arc::default(), // MaxSlots
-            None,
-        );
-
-        let shred = Shred::new_from_data(0, 0, 0, None, true, true, 0, 0x20, 0);
-        // it should send this over the sockets.
-        retransmit_sender.send(vec![shred]).unwrap();
-        let mut packet_batch = PacketBatch::new(vec![]);
-        solana_streamer::packet::recv_from(&mut packet_batch, &me_retransmit, 1).unwrap();
-        assert_eq!(packet_batch.packets.len(), 1);
-        assert!(!packet_batch.packets[0].meta.repair());
-    }
+    use super::*;
 
     #[test]
     fn test_already_received() {