hides implementation details of vote-accounts from public interface (#24087) (#24102)

(cherry picked from commit ef3e3dce7a)

Co-authored-by: behzad nouri <behzadnouri@gmail.com>

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,9 @@
 #### Problem
 
+
+
 #### Summary of Changes
 
+
+
 Fixes #

.github/workflows/explorer_preview.yml

@@ -17,8 +17,7 @@ jobs:
           vercel-token: ${{ secrets.VERCEL_TOKEN }} # Required
           github-token: ${{ secrets.GITHUB_TOKEN }} #Optional 
           vercel-org-id: ${{ secrets.ORG_ID}}  #Required
-          vercel-project-id: ${{ secrets.PROJECT_ID}} #Required 
-          working-directory: ./explorer
+          vercel-project-id: ${{ secrets.PROJECT_ID}} #Required
           scope: ${{ secrets.TEAM_ID }}
      
       - name: vercel url
@@ -36,17 +35,24 @@ jobs:
               #filtered_url=$(cat vercelfile2.txt )
               #echo "$filtered_url" >> .env.preview1 
 
-
-      - name: Run tests
+      - name: Fetching Vercel Preview Deployment Link
         uses: mathiasvr/command-output@v1
-        id: tests2
+        id: test1
         with:
           run: |
                echo "$(cat .env.preview1)"
+      - name: Fetching PR commit URL
+        uses: mathiasvr/command-output@v1
+        id: test2
+        with:
+          run: |
+               HEAD_SHA=$(curl -H "Accept: application/vnd.github.v3+json"  https://api.github.com/repos/solana-labs/solana/pulls | jq .[0] | jq -r .head.sha)
+               USER_NAME=$(curl -H "Accept: application/vnd.github.v3+json"  https://api.github.com/repos/solana-labs/solana/pulls | jq .[0] | jq -r .head.user.login)
+               echo "github.com/$USER_NAME/solana/commit/$HEAD_SHA"
       
-      - name: Slack Notification1
+      - name: Slack Notification4
         uses: rtCamp/action-slack-notify@master
         env:
-         SLACK_MESSAGE: ${{ steps.tests2.outputs.stdout }}
-         SLACK_TITLE: Vercel "Explorer" Preview Deployment Link
+         SLACK_MESSAGE: ' Vercel Link: ${{ steps.test1.outputs.stdout }} PR Commit: ${{steps.test2.outputs.stdout}}'
+         SLACK_TITLE: Vercel "Explorer" Preview Deployment Link , PR Commit
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

.github/workflows/explorer_production.yml

@@ -30,7 +30,8 @@ jobs:
       runs-on: ubuntu-latest
       defaults:
         run:
-          working-directory: explorer
+            working-directory: explorer
+    
       steps:
        - uses: actions/checkout@v2
          with:
@@ -38,9 +39,8 @@ jobs:
        - uses: amondnet/vercel-action@v20
          with:
            vercel-token: ${{ secrets.VERCEL_TOKEN }} # Required
-           github-token: ${{ secrets.GITHUB_TOKEN }} #Optional 
+           github-token: ${{ secrets.PAT }} #Optional 
            vercel-args: '--prod' #for production
            vercel-org-id: ${{ secrets.ORG_ID}}  #Required
            vercel-project-id: ${{ secrets.PROJECT_ID}} #Required 
-           working-directory: ./explorer
            scope: ${{ secrets.TEAM_ID }}

.mergify.yml

@@ -93,6 +93,7 @@ pull_request_rules:
       - author=mergify[bot]
       - head~=^mergify/bp/
       - "#status-failure=0"
+      - "-merged"
     actions:
       label:
         add:

Cargo.toml

@@ -1,63 +1,68 @@
 [workspace]
 members = [
-    "accountsdb-plugin-interface",
-    "accountsdb-plugin-manager",
-    "accounts-cluster-bench",
-    "bench-streamer",
-    "bench-tps",
+    "account-decoder",
     "accounts-bench",
+    "accounts-cluster-bench",
     "banking-bench",
     "banks-client",
     "banks-interface",
     "banks-server",
-    "bucket_map",
+    "bench-streamer",
+    "bench-tps",
     "bloom",
+    "bucket_map",
     "clap-utils",
+    "cli",
     "cli-config",
     "cli-output",
     "client",
+    "client-test",
     "core",
     "dos",
     "download-utils",
     "entry",
     "faucet",
     "frozen-abi",
-    "perf",
-    "validator",
     "genesis",
     "genesis-utils",
+    "geyser-plugin-interface",
+    "geyser-plugin-manager",
     "gossip",
     "install",
     "keygen",
     "ledger",
     "ledger-tool",
     "local-cluster",
-    "logger",
     "log-analyzer",
+    "logger",
+    "measure",
     "merkle-root-bench",
     "merkle-tree",
-    "storage-bigtable",
-    "storage-proto",
-    "streamer",
-    "measure",
     "metrics",
     "net-shaper",
+    "net-utils",
     "notifier",
+    "perf",
     "poh",
     "poh-bench",
     "program-test",
     "programs/address-lookup-table",
     "programs/address-lookup-table-tests",
-    "programs/ed25519-tests",
     "programs/bpf_loader",
     "programs/bpf_loader/gen-syscall-list",
     "programs/compute-budget",
     "programs/config",
+    "programs/ed25519-tests",
     "programs/stake",
     "programs/vote",
+    "programs/zk-token-proof",
+    "rayon-threadlimit",
     "rbpf-cli",
     "remote-wallet",
+    "replica-lib",
+    "replica-node",
     "rpc",
+    "rpc-test",
     "runtime",
     "runtime/store-tool",
     "sdk",
@@ -65,24 +70,19 @@ members = [
     "sdk/cargo-test-bpf",
     "send-transaction-service",
     "stake-accounts",
+    "storage-bigtable",
+    "storage-proto",
+    "streamer",
     "sys-tuner",
+    "test-validator",
     "tokens",
     "transaction-dos",
     "transaction-status",
-    "account-decoder",
     "upload-perf",
-    "net-utils",
+    "validator",
     "version",
-    "cli",
-    "rayon-threadlimit",
     "watchtower",
-    "replica-node",
-    "replica-lib",
-    "test-validator",
-    "rpc-test",
-    "client-test",
     "zk-token-sdk",
-    "programs/zk-token-proof",
 ]
 
 exclude = [

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.10.0"
+version = "1.10.6"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,21 +10,21 @@ license = "Apache-2.0"
 edition = "2021"
 
 [dependencies]
-base64 = "0.12.3"
+Inflector = "0.11.4"
+base64 = "0.13.0"
 bincode = "1.3.3"
 bs58 = "0.4.0"
 bv = "0.11.1"
-Inflector = "0.11.4"
 lazy_static = "1.4.0"
 serde = "1.0.136"
 serde_derive = "1.0.103"
-serde_json = "1.0.78"
-solana-config-program = { path = "../programs/config", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.10.0" }
+serde_json = "1.0.79"
+solana-config-program = { path = "../programs/config", version = "=1.10.6" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
+solana-vote-program = { path = "../programs/vote", version = "=1.10.6" }
 spl-token = { version = "=3.2.0", features = ["no-entrypoint"] }
 thiserror = "1.0"
-zstd = "0.10.0"
+zstd = "0.11.1"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

account-decoder/src/lib.rs

@@ -33,7 +33,7 @@ pub type StringDecimals = String;
 pub const MAX_BASE58_BYTES: usize = 128;
 
 /// A duplicate representation of an Account for pretty JSON serialization
-#[derive(Serialize, Deserialize, Clone, Debug)]
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct UiAccount {
     pub lamports: u64,

accounts-bench/Cargo.toml

@@ -2,21 +2,21 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-accounts-bench"
-version = "1.10.0"
+version = "1.10.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 publish = false
 
 [dependencies]
+clap = "2.33.1"
 log = "0.4.14"
 rayon = "1.5.1"
-solana-logger = { path = "../logger", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-version = { path = "../version", version = "=1.10.0" }
-clap = "2.33.1"
+solana-logger = { path = "../logger", version = "=1.10.6" }
+solana-measure = { path = "../measure", version = "=1.10.6" }
+solana-runtime = { path = "../runtime", version = "=1.10.6" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
+solana-version = { path = "../version", version = "=1.10.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accounts-cluster-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-accounts-cluster-bench"
-version = "1.10.0"
+version = "1.10.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,25 +13,25 @@ clap = "2.33.1"
 log = "0.4.14"
 rand = "0.7.0"
 rayon = "1.5.1"
-solana-account-decoder = { path = "../account-decoder", version = "=1.10.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.10.0" }
-solana-client = { path = "../client", version = "=1.10.0" }
-solana-faucet = { path = "../faucet", version = "=1.10.0" }
-solana-gossip = { path = "../gossip", version = "=1.10.0" }
-solana-logger = { path = "../logger", version = "=1.10.0" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-streamer = { path = "../streamer", version = "=1.10.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
-solana-version = { path = "../version", version = "=1.10.0" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.10.6" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.10.6" }
+solana-client = { path = "../client", version = "=1.10.6" }
+solana-faucet = { path = "../faucet", version = "=1.10.6" }
+solana-gossip = { path = "../gossip", version = "=1.10.6" }
+solana-logger = { path = "../logger", version = "=1.10.6" }
+solana-measure = { path = "../measure", version = "=1.10.6" }
+solana-net-utils = { path = "../net-utils", version = "=1.10.6" }
+solana-runtime = { path = "../runtime", version = "=1.10.6" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
+solana-streamer = { path = "../streamer", version = "=1.10.6" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.6" }
+solana-version = { path = "../version", version = "=1.10.6" }
 spl-token = { version = "=3.2.0", features = ["no-entrypoint"] }
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "=1.10.0" }
-solana-local-cluster = { path = "../local-cluster", version = "=1.10.0" }
-solana-test-validator = { path = "../test-validator", version = "=1.10.0" }
+solana-core = { path = "../core", version = "=1.10.6" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.10.6" }
+solana-test-validator = { path = "../test-validator", version = "=1.10.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-interface/README.md

@@ -1,20 +0,0 @@
-<p align="center">
-  <a href="https://solana.com">
-    <img alt="Solana" src="https://i.imgur.com/IKyzQ6T.png" width="250" />
-  </a>
-</p>
-
-# Solana AccountsDb Plugin Interface
-
-This crate enables an AccountsDb plugin to be plugged into the Solana Validator runtime to take actions
-at the time of each account update; for example, saving the account state to an external database. The plugin must implement the `AccountsDbPlugin` trait. Please see the detail of the `accountsdb_plugin_interface.rs` for the interface definition.
-
-The plugin should produce a `cdylib` dynamic library, which must expose a `C` function `_create_plugin()` that
-instantiates the implementation of the interface.
-
-The `solana-accountsdb-plugin-postgres` crate provides an example of how to create a plugin which saves the accounts data into an
-external PostgreSQL databases.
-
-More information about Solana is available in the [Solana documentation](https://docs.solana.com/).
-
-Still have questions?  Ask us on [Discord](https://discordapp.com/invite/pquxPsq)

accountsdb-plugin-interface/src/lib.rs

@@ -1 +0,0 @@
-pub mod accountsdb_plugin_interface;

accountsdb-plugin-manager/Cargo.toml

@@ -1,29 +0,0 @@
-[package]
-authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2021"
-name = "solana-accountsdb-plugin-manager"
-description = "The Solana AccountsDb plugin manager."
-version = "1.10.0"
-repository = "https://github.com/solana-labs/solana"
-license = "Apache-2.0"
-homepage = "https://solana.com/"
-documentation = "https://docs.rs/solana-validator"
-
-[dependencies]
-bs58 = "0.4.0"
-crossbeam-channel = "0.5"
-json5 = "0.4.1"
-libloading = "0.7.3"
-log = "0.4.11"
-serde_json = "1.0.78"
-solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.10.0" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-solana-metrics = { path = "../metrics", version = "=1.10.0" }
-solana-rpc = { path = "../rpc", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
-thiserror = "1.0.30"
-
-[package.metadata.docs.rs]
-targets = ["x86_64-unknown-linux-gnu"]

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-banking-bench"
-version = "1.10.0"
+version = "1.10.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,17 +14,17 @@ crossbeam-channel = "0.5"
 log = "0.4.14"
 rand = "0.7.0"
 rayon = "1.5.1"
-solana-core = { path = "../core", version = "=1.10.0" }
-solana-gossip = { path = "../gossip", version = "=1.10.0" }
-solana-ledger = { path = "../ledger", version = "=1.10.0" }
-solana-logger = { path = "../logger", version = "=1.10.0" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-solana-perf = { path = "../perf", version = "=1.10.0" }
-solana-poh = { path = "../poh", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-streamer = { path = "../streamer", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-version = { path = "../version", version = "=1.10.0" }
+solana-core = { path = "../core", version = "=1.10.6" }
+solana-gossip = { path = "../gossip", version = "=1.10.6" }
+solana-ledger = { path = "../ledger", version = "=1.10.6" }
+solana-logger = { path = "../logger", version = "=1.10.6" }
+solana-measure = { path = "../measure", version = "=1.10.6" }
+solana-perf = { path = "../perf", version = "=1.10.6" }
+solana-poh = { path = "../poh", version = "=1.10.6" }
+solana-runtime = { path = "../runtime", version = "=1.10.6" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
+solana-streamer = { path = "../streamer", version = "=1.10.6" }
+solana-version = { path = "../version", version = "=1.10.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banks-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-client"
-version = "1.10.0"
+version = "1.10.6"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,17 +12,17 @@ edition = "2021"
 [dependencies]
 borsh = "0.9.3"
 futures = "0.3"
-solana-banks-interface = { path = "../banks-interface", version = "=1.10.0" }
-solana-program = { path = "../sdk/program", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-banks-interface = { path = "../banks-interface", version = "=1.10.6" }
+solana-program = { path = "../sdk/program", version = "=1.10.6" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
 tarpc = { version = "0.27.2", features = ["full"] }
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-banks-server = { path = "../banks-server", version = "=1.10.0" }
+solana-banks-server = { path = "../banks-server", version = "=1.10.6" }
+solana-runtime = { path = "../runtime", version = "=1.10.6" }
 
 [lib]
 crate-type = ["lib"]

banks-interface/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-interface"
-version = "1.10.0"
+version = "1.10.6"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,7 +11,7 @@ edition = "2021"
 
 [dependencies]
 serde = { version = "1.0.136", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
 tarpc = { version = "0.27.2", features = ["full"] }
 
 [lib]

banks-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-server"
-version = "1.10.0"
+version = "1.10.6"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,10 +13,10 @@ edition = "2021"
 bincode = "1.3.3"
 crossbeam-channel = "0.5"
 futures = "0.3"
-solana-banks-interface = { path = "../banks-interface", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.10.0" }
+solana-banks-interface = { path = "../banks-interface", version = "=1.10.6" }
+solana-runtime = { path = "../runtime", version = "=1.10.6" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
+solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.10.6" }
 tarpc = { version = "0.27.2", features = ["full"] }
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }

banks-server/src/banks_server.rs

@@ -24,7 +24,7 @@ use {
         transaction::{self, SanitizedTransaction, Transaction},
     },
     solana_send_transaction_service::{
-        send_transaction_service::{SendTransactionService, TransactionInfo},
+        send_transaction_service::{SendTransactionService, TransactionInfo, DEFAULT_TPU_USE_QUIC},
         tpu_info::NullTpuInfo,
     },
     std::{
@@ -399,6 +399,7 @@ pub async fn start_tcp_server(
                 receiver,
                 5_000,
                 0,
+                DEFAULT_TPU_USE_QUIC,
             );
 
             let server = BanksServer::new(

bench-streamer/Cargo.toml

@@ -2,18 +2,18 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-bench-streamer"
-version = "1.10.0"
+version = "1.10.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 publish = false
 
 [dependencies]
-crossbeam-channel = "0.5"
 clap = "2.33.1"
-solana-streamer = { path = "../streamer", version = "=1.10.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
-solana-version = { path = "../version", version = "=1.10.0" }
+crossbeam-channel = "0.5"
+solana-net-utils = { path = "../net-utils", version = "=1.10.6" }
+solana-streamer = { path = "../streamer", version = "=1.10.6" }
+solana-version = { path = "../version", version = "=1.10.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-bench-tps"
-version = "1.10.0"
+version = "1.10.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,25 +13,25 @@ clap = "2.33.1"
 crossbeam-channel = "0.5"
 log = "0.4.14"
 rayon = "1.5.1"
-serde_json = "1.0.78"
+serde_json = "1.0.79"
 serde_yaml = "0.8.23"
-solana-core = { path = "../core", version = "=1.10.0" }
-solana-genesis = { path = "../genesis", version = "=1.10.0" }
-solana-client = { path = "../client", version = "=1.10.0" }
-solana-faucet = { path = "../faucet", version = "=1.10.0" }
-solana-gossip = { path = "../gossip", version = "=1.10.0" }
-solana-logger = { path = "../logger", version = "=1.10.0" }
-solana-metrics = { path = "../metrics", version = "=1.10.0" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-streamer = { path = "../streamer", version = "=1.10.0" }
-solana-version = { path = "../version", version = "=1.10.0" }
+solana-client = { path = "../client", version = "=1.10.6" }
+solana-core = { path = "../core", version = "=1.10.6" }
+solana-faucet = { path = "../faucet", version = "=1.10.6" }
+solana-genesis = { path = "../genesis", version = "=1.10.6" }
+solana-gossip = { path = "../gossip", version = "=1.10.6" }
+solana-logger = { path = "../logger", version = "=1.10.6" }
+solana-measure = { path = "../measure", version = "=1.10.6" }
+solana-metrics = { path = "../metrics", version = "=1.10.6" }
+solana-net-utils = { path = "../net-utils", version = "=1.10.6" }
+solana-runtime = { path = "../runtime", version = "=1.10.6" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
+solana-streamer = { path = "../streamer", version = "=1.10.6" }
+solana-version = { path = "../version", version = "=1.10.6" }
 
 [dev-dependencies]
-serial_test = "0.5.1"
-solana-local-cluster = { path = "../local-cluster", version = "=1.10.0" }
+serial_test = "0.6.0"
+solana-local-cluster = { path = "../local-cluster", version = "=1.10.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/src/bench.rs

@@ -475,6 +475,7 @@ fn do_tx_transfers<T: Client>(
             let tx_len = txs0.len();
             let transfer_start = Instant::now();
             let mut old_transactions = false;
+            let mut transactions = Vec::<_>::new();
             for tx in txs0 {
                 let now = timestamp();
                 // Transactions that are too old will be rejected by the cluster Don't bother
@@ -483,10 +484,13 @@ fn do_tx_transfers<T: Client>(
                     old_transactions = true;
                     continue;
                 }
-                client
-                    .async_send_transaction(tx.0)
-                    .expect("async_send_transaction in do_tx_transfers");
+                transactions.push(tx.0);
             }
+
+            if let Err(error) = client.async_send_batch(transactions) {
+                warn!("send_batch_sync in do_tx_transfers failed: {}", error);
+            }
+
             if old_transactions {
                 let mut shared_txs_wl = shared_txs.write().expect("write lock in do_tx_transfers");
                 shared_txs_wl.clear();

bloom/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bloom"
-version = "1.10.0"
+version = "1.10.6"
 description = "Solana bloom filter"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,14 +12,14 @@ edition = "2021"
 [dependencies]
 bv = { version = "0.11.1", features = ["serde"] }
 fnv = "1.0.7"
-rand = "0.7.0"
-serde = { version = "1.0.136", features = ["rc"] }
-rayon = "1.5.1"
-serde_derive = "1.0.103"
-solana-frozen-abi = { path = "../frozen-abi", version = "=1.10.0" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
 log = "0.4.14"
+rand = "0.7.0"
+rayon = "1.5.1"
+serde = { version = "1.0.136", features = ["rc"] }
+serde_derive = "1.0.103"
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.10.6" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.10.6" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
 
 [lib]
 crate-type = ["lib"]

bucket_map/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bucket-map"
-version = "1.10.0"
+version = "1.10.6"
 description = "solana-bucket-map"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-bucket-map"
@@ -11,18 +11,18 @@ license = "Apache-2.0"
 edition = "2021"
 
 [dependencies]
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-memmap2 = "0.5.2"
 log = { version = "0.4.11" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-rand = "0.7.0"
-tempfile = "3.3.0"
+memmap2 = "0.5.3"
 modular-bitfield = "0.11.2"
+rand = "0.7.0"
+solana-measure = { path = "../measure", version = "=1.10.6" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
+tempfile = "3.3.0"
 
 [dev-dependencies]
 fs_extra = "1.2.0"
 rayon = "1.5.0"
-solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-logger = { path = "../logger", version = "=1.10.6" }
 
 [lib]
 crate-type = ["lib"]

ci/buildkite-pipeline-in-disk.sh

@@ -137,7 +137,7 @@ all_test_steps() {
              ^ci/test-coverage.sh \
              ^scripts/coverage.sh \
       ; then
-    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 40
+    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 50
     wait_step
   else
     annotate --style info --context test-coverage \
@@ -152,14 +152,14 @@ all_test_steps() {
              ^ci/test-coverage.sh \
              ^scripts/coverage-in-disk.sh \
       ; then
-    command_step coverage-in-disk ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 40
+    command_step coverage-in-disk ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 50
     wait_step
   else
     annotate --style info --context test-coverage \
       "Coverage skipped as no .rs files were modified"
   fi
   # Full test suite
-  command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 60
+  command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 70
   wait_step
 
   # BPF test suite

ci/buildkite-pipeline.sh

@@ -102,6 +102,8 @@ command_step() {
     command: "$2"
     timeout_in_minutes: $3
     artifact_paths: "log-*.txt"
+    agents:
+      - "queue=solana"
 EOF
 }
 
@@ -137,7 +139,7 @@ all_test_steps() {
              ^ci/test-coverage.sh \
              ^scripts/coverage.sh \
       ; then
-    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 40
+    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 50
     wait_step
   else
     annotate --style info --context test-coverage \
@@ -145,7 +147,7 @@ all_test_steps() {
   fi
 
   # Full test suite
-  command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 60
+  command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 70
   wait_step
 
   # BPF test suite
@@ -168,7 +170,7 @@ all_test_steps() {
     timeout_in_minutes: 20
     artifact_paths: "bpf-dumps.tar.bz2"
     agents:
-      - "queue=default"
+      - "queue=solana"
 EOF
   else
     annotate --style info \
@@ -221,6 +223,8 @@ EOF
   - command: "scripts/build-downstream-projects.sh"
     name: "downstream-projects"
     timeout_in_minutes: 30
+    agents:
+      - "queue=solana"
 EOF
   else
     annotate --style info \
@@ -246,6 +250,8 @@ EOF
   - command: "scripts/build-downstream-anchor-projects.sh"
     name: "downstream-anchor-projects"
     timeout_in_minutes: 10
+    agents:
+      - "queue=solana"
 EOF
   else
     annotate --style info \

ci/do-audit.sh

@@ -8,11 +8,6 @@ src_root="$(readlink -f "${here}/..")"
 cd "${src_root}"
 
 cargo_audit_ignores=(
-  # failure is officially deprecated/unmaintained
-  #
-  # Blocked on multiple upstream crates removing their `failure` dependency.
-  --ignore RUSTSEC-2020-0036
-
   # `net2` crate has been deprecated; use `socket2` instead
   #
   # Blocked on https://github.com/paritytech/jsonrpc/issues/575
@@ -30,22 +25,10 @@ cargo_audit_ignores=(
 
   # generic-array: arr! macro erases lifetimes
   #
-  # Blocked on libsecp256k1 releasing with upgraded dependencies
-  # https://github.com/paritytech/libsecp256k1/issues/66
+  # Blocked on new spl dependencies on solana-program v1.9
+  # due to curve25519-dalek dependency
   --ignore RUSTSEC-2020-0146
 
-  # hyper: Lenient `hyper` header parsing of `Content-Length` could allow request smuggling
-  #
-  # Blocked on jsonrpc removing dependency on unmaintained `websocket`
-  # https://github.com/paritytech/jsonrpc/issues/605
-  --ignore RUSTSEC-2021-0078
-
-  # hyper: Integer overflow in `hyper`'s parsing of the `Transfer-Encoding` header leads to data loss
-  #
-  # Blocked on jsonrpc removing dependency on unmaintained `websocket`
-  # https://github.com/paritytech/jsonrpc/issues/605
-  --ignore RUSTSEC-2021-0079
-
   # chrono: Potential segfault in `localtime_r` invocations
   #
   # Blocked due to no safe upgrade

ci/publish-crate.sh

@@ -70,7 +70,7 @@ for Cargo_toml in $Cargo_tomls; do
         rm -rf crate-test
         "$cargo" stable init crate-test
         cd crate-test/
-        echo "${crate_name} = \"${expectedCrateVersion}\"" >> Cargo.toml
+        echo "${crate_name} = \"=${expectedCrateVersion}\"" >> Cargo.toml
         echo "[workspace]" >> Cargo.toml
         "$cargo" stable check
       ) && really_uploaded=1

ci/test-checks.sh

@@ -57,26 +57,20 @@ if [[ $CI_BASE_BRANCH = "$EDGE_CHANNEL" ]]; then
     exit "$check_status"
   fi
 
-  # Ensure nightly and --benches
+   # Ensure nightly and --benches
   _ scripts/cargo-for-all-lock-files.sh nightly check --locked --all-targets
 else
   echo "Note: cargo-for-all-lock-files.sh skipped because $CI_BASE_BRANCH != $EDGE_CHANNEL"
 fi
 
-_ ci/order-crates-for-publishing.py
+ _ ci/order-crates-for-publishing.py
 
 # -Z... is needed because of clippy bug: https://github.com/rust-lang/rust-clippy/issues/4612
 # run nightly clippy for `sdk/` as there's a moderate amount of nightly-only code there
-_ "$cargo" nightly clippy -Zunstable-options --workspace --all-targets -- --deny=warnings --deny=clippy::integer_arithmetic
+ _ scripts/cargo-for-all-lock-files.sh -- nightly clippy -Zunstable-options --all-targets -- --deny=warnings --deny=clippy::integer_arithmetic
 
-_ "$cargo" nightly fmt --all -- --check
+_ scripts/cargo-for-all-lock-files.sh -- nightly fmt --all -- --check
 
-_ ci/do-audit.sh
-
-{
-  cd programs/bpf
-  _ "$cargo" nightly clippy --all -- --deny=warnings --allow=clippy::missing_safety_doc
-  _ "$cargo" nightly fmt --all -- --check
-}
+ _ ci/do-audit.sh
 
 echo --- ok

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.10.0"
+version = "1.10.6"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,16 +10,16 @@ documentation = "https://docs.rs/solana-clap-utils"
 edition = "2021"
 
 [dependencies]
+chrono = "0.4"
 clap = "2.33.0"
-rpassword = "5.0"
-solana-perf = { path = "../perf", version = "=1.10.0" }
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.10.0", default-features = false}
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
+rpassword = "6.0"
+solana-perf = { path = "../perf", version = "=1.10.6" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.10.6", default-features = false }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
 thiserror = "1.0.30"
 tiny-bip39 = "0.8.2"
 uriparse = "0.6.3"
 url = "2.2.2"
-chrono = "0.4"
 
 [dev-dependencies]
 tempfile = "3.3.0"

clap-utils/src/keypair.rs

@@ -17,7 +17,7 @@ use {
     },
     bip39::{Language, Mnemonic, Seed},
     clap::ArgMatches,
-    rpassword::prompt_password_stderr,
+    rpassword::prompt_password,
     solana_remote_wallet::{
         locator::{Locator as RemoteWalletLocator, LocatorError as RemoteWalletLocatorError},
         remote_keypair::generate_remote_keypair,
@@ -945,9 +945,9 @@ pub const SKIP_SEED_PHRASE_VALIDATION_ARG: ArgConstant<'static> = ArgConstant {
 
 /// Prompts user for a passphrase and then asks for confirmirmation to check for mistakes
 pub fn prompt_passphrase(prompt: &str) -> Result<String, Box<dyn error::Error>> {
-    let passphrase = prompt_password_stderr(prompt)?;
+    let passphrase = prompt_password(prompt)?;
     if !passphrase.is_empty() {
-        let confirmed = rpassword::prompt_password_stderr("Enter same passphrase again: ")?;
+        let confirmed = rpassword::prompt_password("Enter same passphrase again: ")?;
         if confirmed != passphrase {
             return Err("Passphrases did not match".into());
         }
@@ -1055,7 +1055,7 @@ pub fn keypair_from_seed_phrase(
     derivation_path: Option<DerivationPath>,
     legacy: bool,
 ) -> Result<Keypair, Box<dyn error::Error>> {
-    let seed_phrase = prompt_password_stderr(&format!("[{}] seed phrase: ", keypair_name))?;
+    let seed_phrase = prompt_password(&format!("[{}] seed phrase: ", keypair_name))?;
     let seed_phrase = seed_phrase.trim();
     let passphrase_prompt = format!(
         "[{}] If this seed phrase has an associated passphrase, enter it now. Otherwise, press ENTER to continue: ",

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.10.0"
+version = "1.10.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,7 +18,7 @@ serde_yaml = "0.8.23"
 url = "2.2.2"
 
 [dev-dependencies]
-anyhow = "1.0.53"
+anyhow = "1.0.56"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

cli-output/Cargo.toml

@@ -3,29 +3,32 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.10.0"
+version = "1.10.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-cli-output"
 
 [dependencies]
+Inflector = "0.11.4"
 base64 = "0.13.0"
 chrono = { version = "0.4.11", features = ["serde"] }
 clap = "2.33.0"
 console = "0.15.0"
 humantime = "2.0.1"
-Inflector = "0.11.4"
 indicatif = "0.16.2"
 serde = "1.0.136"
-serde_json = "1.0.78"
-solana-account-decoder = { path = "../account-decoder", version = "=1.10.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.10.0" }
-solana-client = { path = "../client", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.10.0" }
+serde_json = "1.0.79"
+solana-account-decoder = { path = "../account-decoder", version = "=1.10.6" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.10.6" }
+solana-client = { path = "../client", version = "=1.10.6" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.6" }
+solana-vote-program = { path = "../programs/vote", version = "=1.10.6" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 
+[dev-dependencies]
+ed25519-dalek = "=1.0.1"
+
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

cli-output/src/cli_output.rs

@@ -28,7 +28,7 @@ use {
         signature::Signature,
         stake::state::{Authorized, Lockup},
         stake_history::StakeHistoryEntry,
-        transaction::{Transaction, TransactionError},
+        transaction::{Transaction, TransactionError, VersionedTransaction},
     },
     solana_transaction_status::{
         EncodedConfirmedBlock, EncodedTransaction, TransactionConfirmationStatus,
@@ -2218,7 +2218,7 @@ pub enum CliSignatureVerificationStatus {
 }
 
 impl CliSignatureVerificationStatus {
-    pub fn verify_transaction(tx: &Transaction) -> Vec<Self> {
+    pub fn verify_transaction(tx: &VersionedTransaction) -> Vec<Self> {
         tx.verify_with_results()
             .iter()
             .zip(&tx.signatures)
@@ -2335,7 +2335,7 @@ impl fmt::Display for CliBlock {
             writeln_transaction(
                 f,
                 &transaction_with_meta.transaction.decode().unwrap(),
-                &transaction_with_meta.meta,
+                transaction_with_meta.meta.as_ref(),
                 "  ",
                 None,
                 None,
@@ -2354,7 +2354,7 @@ pub struct CliTransaction {
     #[serde(skip_serializing)]
     pub slot: Option<Slot>,
     #[serde(skip_serializing)]
-    pub decoded_transaction: Transaction,
+    pub decoded_transaction: VersionedTransaction,
     #[serde(skip_serializing)]
     pub prefix: String,
     #[serde(skip_serializing_if = "Vec::is_empty")]
@@ -2369,7 +2369,7 @@ impl fmt::Display for CliTransaction {
         writeln_transaction(
             f,
             &self.decoded_transaction,
-            &self.meta,
+            self.meta.as_ref(),
             &self.prefix,
             if !self.sigverify_status.is_empty() {
                 Some(&self.sigverify_status)
@@ -2776,10 +2776,10 @@ mod tests {
 
         let expected_msg = "AwECBwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDgTl3Dqh9\
             F19Wo1Rmw0x+zMuNipG07jeiXfYPW4/Js5QEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE\
-            BAQEBAYGBgYGBgYGBgYGBgYGBgYGBgYGBgYGBgYGBgYGBgYGBQUFBQUFBQUFBQUFBQUFBQUF\
-            BQUFBQUFBQUFBQUFBQUGp9UXGSxWjuCKhF9z0peIzwNcMUWyGrNE2AYuqUAAAAAAAAAAAAAA\
-            AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcH\
-            BwcCBgMDBQIEBAAAAAYCAQQMAgAAACoAAAAAAAAA"
+            BAQEBAUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBgYGBgYGBgYGBgYGBgYGBgYG\
+            BgYGBgYGBgYGBgYGBgYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAan1RcZLFaO\
+            4IqEX3PSl4jPA1wxRbIas0TYBi6pQAAABwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcH\
+            BwcCBQMEBgIEBAAAAAUCAQMMAgAAACoAAAAAAAAA"
             .to_string();
         let config = ReturnSignersConfig {
             dump_transaction_message: true,

cli-output/src/display.rs

@@ -4,10 +4,18 @@ use {
     console::style,
     indicatif::{ProgressBar, ProgressStyle},
     solana_sdk::{
-        clock::UnixTimestamp, hash::Hash, message::Message, native_token::lamports_to_sol,
-        program_utils::limited_deserialize, pubkey::Pubkey, stake, transaction::Transaction,
+        clock::UnixTimestamp,
+        hash::Hash,
+        instruction::CompiledInstruction,
+        message::v0::MessageAddressTableLookup,
+        native_token::lamports_to_sol,
+        program_utils::limited_deserialize,
+        pubkey::Pubkey,
+        signature::Signature,
+        stake,
+        transaction::{TransactionError, TransactionVersion, VersionedTransaction},
     },
-    solana_transaction_status::UiTransactionStatusMeta,
+    solana_transaction_status::{Rewards, UiTransactionStatusMeta},
     spl_memo::{id as spl_memo_id, v1::id as spl_memo_v1_id},
     std::{collections::HashMap, fmt, io},
 };
@@ -131,22 +139,28 @@ pub fn println_signers(
     println!();
 }
 
-fn format_account_mode(message: &Message, index: usize) -> String {
+struct CliAccountMeta {
+    is_signer: bool,
+    is_writable: bool,
+    is_invoked: bool,
+}
+
+fn format_account_mode(meta: CliAccountMeta) -> String {
     format!(
         "{}r{}{}", // accounts are always readable...
-        if message.is_signer(index) {
+        if meta.is_signer {
             "s" // stands for signer
         } else {
             "-"
         },
-        if message.is_writable(index) {
+        if meta.is_writable {
             "w" // comment for consistent rust fmt (no joking; lol)
         } else {
             "-"
         },
         // account may be executable on-chain while not being
         // designated as a program-id in the message
-        if message.maybe_executable(index) {
+        if meta.is_invoked {
             "x"
         } else {
             // programs to be executed via CPI cannot be identified as
@@ -156,202 +170,83 @@ fn format_account_mode(message: &Message, index: usize) -> String {
     )
 }
 
-pub fn write_transaction<W: io::Write>(
+fn write_transaction<W: io::Write>(
     w: &mut W,
-    transaction: &Transaction,
-    transaction_status: &Option<UiTransactionStatusMeta>,
+    transaction: &VersionedTransaction,
+    transaction_status: Option<&UiTransactionStatusMeta>,
     prefix: &str,
     sigverify_status: Option<&[CliSignatureVerificationStatus]>,
     block_time: Option<UnixTimestamp>,
+    timezone: CliTimezone,
 ) -> io::Result<()> {
+    write_block_time(w, block_time, timezone, prefix)?;
+
     let message = &transaction.message;
-    if let Some(block_time) = block_time {
-        writeln!(
-            w,
-            "{}Block Time: {:?}",
-            prefix,
-            Local.timestamp(block_time, 0)
-        )?;
-    }
-    writeln!(
-        w,
-        "{}Recent Blockhash: {:?}",
-        prefix, message.recent_blockhash
-    )?;
-    let sigverify_statuses = if let Some(sigverify_status) = sigverify_status {
-        sigverify_status
+    let account_keys: Vec<AccountKeyType> = {
+        let static_keys_iter = message
+            .static_account_keys()
             .iter()
-            .map(|s| format!(" ({})", s))
-            .collect()
-    } else {
-        vec!["".to_string(); transaction.signatures.len()]
+            .map(AccountKeyType::Known);
+        let dynamic_keys: Vec<AccountKeyType> = message
+            .address_table_lookups()
+            .map(transform_lookups_to_unknown_keys)
+            .unwrap_or_default();
+        static_keys_iter.chain(dynamic_keys).collect()
     };
-    for (signature_index, (signature, sigverify_status)) in transaction
-        .signatures
-        .iter()
-        .zip(&sigverify_statuses)
-        .enumerate()
-    {
-        writeln!(
-            w,
-            "{}Signature {}: {:?}{}",
-            prefix, signature_index, signature, sigverify_status,
-        )?;
-    }
+
+    write_version(w, transaction.version(), prefix)?;
+    write_recent_blockhash(w, message.recent_blockhash(), prefix)?;
+    write_signatures(w, &transaction.signatures, sigverify_status, prefix)?;
+
     let mut fee_payer_index = None;
-    for (account_index, account) in message.account_keys.iter().enumerate() {
+    for (account_index, account) in account_keys.iter().enumerate() {
         if fee_payer_index.is_none() && message.is_non_loader_key(account_index) {
             fee_payer_index = Some(account_index)
         }
-        writeln!(
+
+        let account_meta = CliAccountMeta {
+            is_signer: message.is_signer(account_index),
+            is_writable: message.is_maybe_writable(account_index),
+            is_invoked: message.is_invoked(account_index),
+        };
+
+        write_account(
             w,
-            "{}Account {}: {} {}{}",
-            prefix,
             account_index,
-            format_account_mode(message, account_index),
-            account,
-            if Some(account_index) == fee_payer_index {
-                " (fee payer)"
-            } else {
-                ""
-            },
+            *account,
+            format_account_mode(account_meta),
+            Some(account_index) == fee_payer_index,
+            prefix,
         )?;
     }
-    for (instruction_index, instruction) in message.instructions.iter().enumerate() {
-        let program_pubkey = message.account_keys[instruction.program_id_index as usize];
-        writeln!(w, "{}Instruction {}", prefix, instruction_index)?;
-        writeln!(
+
+    for (instruction_index, instruction) in message.instructions().iter().enumerate() {
+        let program_pubkey = account_keys[instruction.program_id_index as usize];
+        let instruction_accounts = instruction
+            .accounts
+            .iter()
+            .map(|account_index| (account_keys[*account_index as usize], *account_index));
+
+        write_instruction(
             w,
-            "{}  Program:   {} ({})",
-            prefix, program_pubkey, instruction.program_id_index
+            instruction_index,
+            program_pubkey,
+            instruction,
+            instruction_accounts,
+            prefix,
         )?;
-        for (account_index, account) in instruction.accounts.iter().enumerate() {
-            let account_pubkey = message.account_keys[*account as usize];
-            writeln!(
-                w,
-                "{}  Account {}: {} ({})",
-                prefix, account_index, account_pubkey, account
-            )?;
-        }
+    }
 
-        let mut raw = true;
-        if program_pubkey == solana_vote_program::id() {
-            if let Ok(vote_instruction) = limited_deserialize::<
-                solana_vote_program::vote_instruction::VoteInstruction,
-            >(&instruction.data)
-            {
-                writeln!(w, "{}  {:?}", prefix, vote_instruction)?;
-                raw = false;
-            }
-        } else if program_pubkey == stake::program::id() {
-            if let Ok(stake_instruction) =
-                limited_deserialize::<stake::instruction::StakeInstruction>(&instruction.data)
-            {
-                writeln!(w, "{}  {:?}", prefix, stake_instruction)?;
-                raw = false;
-            }
-        } else if program_pubkey == solana_sdk::system_program::id() {
-            if let Ok(system_instruction) = limited_deserialize::<
-                solana_sdk::system_instruction::SystemInstruction,
-            >(&instruction.data)
-            {
-                writeln!(w, "{}  {:?}", prefix, system_instruction)?;
-                raw = false;
-            }
-        } else if is_memo_program(&program_pubkey) {
-            if let Ok(s) = std::str::from_utf8(&instruction.data) {
-                writeln!(w, "{}  Data: \"{}\"", prefix, s)?;
-                raw = false;
-            }
-        }
-
-        if raw {
-            writeln!(w, "{}  Data: {:?}", prefix, instruction.data)?;
-        }
+    if let Some(address_table_lookups) = message.address_table_lookups() {
+        write_address_table_lookups(w, address_table_lookups, prefix)?;
     }
 
     if let Some(transaction_status) = transaction_status {
-        writeln!(
-            w,
-            "{}Status: {}",
-            prefix,
-            match &transaction_status.status {
-                Ok(_) => "Ok".into(),
-                Err(err) => err.to_string(),
-            }
-        )?;
-        writeln!(
-            w,
-            "{}  Fee: ◎{}",
-            prefix,
-            lamports_to_sol(transaction_status.fee)
-        )?;
-        assert_eq!(
-            transaction_status.pre_balances.len(),
-            transaction_status.post_balances.len()
-        );
-        for (i, (pre, post)) in transaction_status
-            .pre_balances
-            .iter()
-            .zip(transaction_status.post_balances.iter())
-            .enumerate()
-        {
-            if pre == post {
-                writeln!(
-                    w,
-                    "{}  Account {} balance: ◎{}",
-                    prefix,
-                    i,
-                    lamports_to_sol(*pre)
-                )?;
-            } else {
-                writeln!(
-                    w,
-                    "{}  Account {} balance: ◎{} -> ◎{}",
-                    prefix,
-                    i,
-                    lamports_to_sol(*pre),
-                    lamports_to_sol(*post)
-                )?;
-            }
-        }
-
-        if let Some(log_messages) = &transaction_status.log_messages {
-            if !log_messages.is_empty() {
-                writeln!(w, "{}Log Messages:", prefix,)?;
-                for log_message in log_messages {
-                    writeln!(w, "{}  {}", prefix, log_message)?;
-                }
-            }
-        }
-
-        if let Some(rewards) = &transaction_status.rewards {
-            if !rewards.is_empty() {
-                writeln!(w, "{}Rewards:", prefix,)?;
-                writeln!(
-                    w,
-                    "{}  {:<44}  {:^15}  {:<15}  {:<20}",
-                    prefix, "Address", "Type", "Amount", "New Balance"
-                )?;
-                for reward in rewards {
-                    let sign = if reward.lamports < 0 { "-" } else { "" };
-                    writeln!(
-                        w,
-                        "{}  {:<44}  {:^15}  {}◎{:<14.9}  ◎{:<18.9}",
-                        prefix,
-                        reward.pubkey,
-                        if let Some(reward_type) = reward.reward_type {
-                            format!("{}", reward_type)
-                        } else {
-                            "-".to_string()
-                        },
-                        sign,
-                        lamports_to_sol(reward.lamports.abs() as u64),
-                        lamports_to_sol(reward.post_balance)
-                    )?;
-                }
-            }
-        }
+        write_status(w, &transaction_status.status, prefix)?;
+        write_fees(w, transaction_status.fee, prefix)?;
+        write_balances(w, transaction_status, prefix)?;
+        write_log_messages(w, transaction_status.log_messages.as_ref(), prefix)?;
+        write_rewards(w, transaction_status.rewards.as_ref(), prefix)?;
     } else {
         writeln!(w, "{}Status: Unavailable", prefix)?;
     }
@@ -359,9 +254,347 @@ pub fn write_transaction<W: io::Write>(
     Ok(())
 }
 
+fn transform_lookups_to_unknown_keys(lookups: &[MessageAddressTableLookup]) -> Vec<AccountKeyType> {
+    let unknown_writable_keys = lookups
+        .iter()
+        .enumerate()
+        .flat_map(|(lookup_index, lookup)| {
+            lookup
+                .writable_indexes
+                .iter()
+                .map(move |table_index| AccountKeyType::Unknown {
+                    lookup_index,
+                    table_index: *table_index,
+                })
+        });
+
+    let unknown_readonly_keys = lookups
+        .iter()
+        .enumerate()
+        .flat_map(|(lookup_index, lookup)| {
+            lookup
+                .readonly_indexes
+                .iter()
+                .map(move |table_index| AccountKeyType::Unknown {
+                    lookup_index,
+                    table_index: *table_index,
+                })
+        });
+
+    unknown_writable_keys.chain(unknown_readonly_keys).collect()
+}
+
+enum CliTimezone {
+    Local,
+    #[allow(dead_code)]
+    Utc,
+}
+
+fn write_block_time<W: io::Write>(
+    w: &mut W,
+    block_time: Option<UnixTimestamp>,
+    timezone: CliTimezone,
+    prefix: &str,
+) -> io::Result<()> {
+    if let Some(block_time) = block_time {
+        let block_time_output = match timezone {
+            CliTimezone::Local => format!("{:?}", Local.timestamp(block_time, 0)),
+            CliTimezone::Utc => format!("{:?}", Utc.timestamp(block_time, 0)),
+        };
+        writeln!(w, "{}Block Time: {}", prefix, block_time_output,)?;
+    }
+    Ok(())
+}
+
+fn write_version<W: io::Write>(
+    w: &mut W,
+    version: TransactionVersion,
+    prefix: &str,
+) -> io::Result<()> {
+    let version = match version {
+        TransactionVersion::Legacy(_) => "legacy".to_string(),
+        TransactionVersion::Number(number) => number.to_string(),
+    };
+    writeln!(w, "{}Version: {}", prefix, version)
+}
+
+fn write_recent_blockhash<W: io::Write>(
+    w: &mut W,
+    recent_blockhash: &Hash,
+    prefix: &str,
+) -> io::Result<()> {
+    writeln!(w, "{}Recent Blockhash: {:?}", prefix, recent_blockhash)
+}
+
+fn write_signatures<W: io::Write>(
+    w: &mut W,
+    signatures: &[Signature],
+    sigverify_status: Option<&[CliSignatureVerificationStatus]>,
+    prefix: &str,
+) -> io::Result<()> {
+    let sigverify_statuses = if let Some(sigverify_status) = sigverify_status {
+        sigverify_status
+            .iter()
+            .map(|s| format!(" ({})", s))
+            .collect()
+    } else {
+        vec!["".to_string(); signatures.len()]
+    };
+    for (signature_index, (signature, sigverify_status)) in
+        signatures.iter().zip(&sigverify_statuses).enumerate()
+    {
+        writeln!(
+            w,
+            "{}Signature {}: {:?}{}",
+            prefix, signature_index, signature, sigverify_status,
+        )?;
+    }
+    Ok(())
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+enum AccountKeyType<'a> {
+    Known(&'a Pubkey),
+    Unknown {
+        lookup_index: usize,
+        table_index: u8,
+    },
+}
+
+impl fmt::Display for AccountKeyType<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Self::Known(address) => write!(f, "{}", address),
+            Self::Unknown {
+                lookup_index,
+                table_index,
+            } => {
+                write!(
+                    f,
+                    "Unknown Address (uses lookup {} and index {})",
+                    lookup_index, table_index
+                )
+            }
+        }
+    }
+}
+
+fn write_account<W: io::Write>(
+    w: &mut W,
+    account_index: usize,
+    account_address: AccountKeyType,
+    account_mode: String,
+    is_fee_payer: bool,
+    prefix: &str,
+) -> io::Result<()> {
+    writeln!(
+        w,
+        "{}Account {}: {} {}{}",
+        prefix,
+        account_index,
+        account_mode,
+        account_address,
+        if is_fee_payer { " (fee payer)" } else { "" },
+    )
+}
+
+fn write_instruction<'a, W: io::Write>(
+    w: &mut W,
+    instruction_index: usize,
+    program_pubkey: AccountKeyType,
+    instruction: &CompiledInstruction,
+    instruction_accounts: impl Iterator<Item = (AccountKeyType<'a>, u8)>,
+    prefix: &str,
+) -> io::Result<()> {
+    writeln!(w, "{}Instruction {}", prefix, instruction_index)?;
+    writeln!(
+        w,
+        "{}  Program:   {} ({})",
+        prefix, program_pubkey, instruction.program_id_index
+    )?;
+    for (index, (account_address, account_index)) in instruction_accounts.enumerate() {
+        writeln!(
+            w,
+            "{}  Account {}: {} ({})",
+            prefix, index, account_address, account_index
+        )?;
+    }
+
+    let mut raw = true;
+    if let AccountKeyType::Known(program_pubkey) = program_pubkey {
+        if program_pubkey == &solana_vote_program::id() {
+            if let Ok(vote_instruction) = limited_deserialize::<
+                solana_vote_program::vote_instruction::VoteInstruction,
+            >(&instruction.data)
+            {
+                writeln!(w, "{}  {:?}", prefix, vote_instruction)?;
+                raw = false;
+            }
+        } else if program_pubkey == &stake::program::id() {
+            if let Ok(stake_instruction) =
+                limited_deserialize::<stake::instruction::StakeInstruction>(&instruction.data)
+            {
+                writeln!(w, "{}  {:?}", prefix, stake_instruction)?;
+                raw = false;
+            }
+        } else if program_pubkey == &solana_sdk::system_program::id() {
+            if let Ok(system_instruction) = limited_deserialize::<
+                solana_sdk::system_instruction::SystemInstruction,
+            >(&instruction.data)
+            {
+                writeln!(w, "{}  {:?}", prefix, system_instruction)?;
+                raw = false;
+            }
+        } else if is_memo_program(program_pubkey) {
+            if let Ok(s) = std::str::from_utf8(&instruction.data) {
+                writeln!(w, "{}  Data: \"{}\"", prefix, s)?;
+                raw = false;
+            }
+        }
+    }
+
+    if raw {
+        writeln!(w, "{}  Data: {:?}", prefix, instruction.data)?;
+    }
+
+    Ok(())
+}
+
+fn write_address_table_lookups<W: io::Write>(
+    w: &mut W,
+    address_table_lookups: &[MessageAddressTableLookup],
+    prefix: &str,
+) -> io::Result<()> {
+    for (lookup_index, lookup) in address_table_lookups.iter().enumerate() {
+        writeln!(w, "{}Address Table Lookup {}", prefix, lookup_index,)?;
+        writeln!(w, "{}  Table Account: {}", prefix, lookup.account_key,)?;
+        writeln!(
+            w,
+            "{}  Writable Indexes: {:?}",
+            prefix,
+            &lookup.writable_indexes[..],
+        )?;
+        writeln!(
+            w,
+            "{}  Readonly Indexes: {:?}",
+            prefix,
+            &lookup.readonly_indexes[..],
+        )?;
+    }
+    Ok(())
+}
+
+fn write_rewards<W: io::Write>(
+    w: &mut W,
+    rewards: Option<&Rewards>,
+    prefix: &str,
+) -> io::Result<()> {
+    if let Some(rewards) = rewards {
+        if !rewards.is_empty() {
+            writeln!(w, "{}Rewards:", prefix,)?;
+            writeln!(
+                w,
+                "{}  {:<44}  {:^15}  {:<16}  {:<20}",
+                prefix, "Address", "Type", "Amount", "New Balance"
+            )?;
+            for reward in rewards {
+                let sign = if reward.lamports < 0 { "-" } else { "" };
+                writeln!(
+                    w,
+                    "{}  {:<44}  {:^15}  {}◎{:<14.9}  ◎{:<18.9}",
+                    prefix,
+                    reward.pubkey,
+                    if let Some(reward_type) = reward.reward_type {
+                        format!("{}", reward_type)
+                    } else {
+                        "-".to_string()
+                    },
+                    sign,
+                    lamports_to_sol(reward.lamports.abs() as u64),
+                    lamports_to_sol(reward.post_balance)
+                )?;
+            }
+        }
+    }
+    Ok(())
+}
+
+fn write_status<W: io::Write>(
+    w: &mut W,
+    transaction_status: &Result<(), TransactionError>,
+    prefix: &str,
+) -> io::Result<()> {
+    writeln!(
+        w,
+        "{}Status: {}",
+        prefix,
+        match transaction_status {
+            Ok(_) => "Ok".into(),
+            Err(err) => err.to_string(),
+        }
+    )
+}
+
+fn write_fees<W: io::Write>(w: &mut W, transaction_fee: u64, prefix: &str) -> io::Result<()> {
+    writeln!(w, "{}  Fee: ◎{}", prefix, lamports_to_sol(transaction_fee))
+}
+
+fn write_balances<W: io::Write>(
+    w: &mut W,
+    transaction_status: &UiTransactionStatusMeta,
+    prefix: &str,
+) -> io::Result<()> {
+    assert_eq!(
+        transaction_status.pre_balances.len(),
+        transaction_status.post_balances.len()
+    );
+    for (i, (pre, post)) in transaction_status
+        .pre_balances
+        .iter()
+        .zip(transaction_status.post_balances.iter())
+        .enumerate()
+    {
+        if pre == post {
+            writeln!(
+                w,
+                "{}  Account {} balance: ◎{}",
+                prefix,
+                i,
+                lamports_to_sol(*pre)
+            )?;
+        } else {
+            writeln!(
+                w,
+                "{}  Account {} balance: ◎{} -> ◎{}",
+                prefix,
+                i,
+                lamports_to_sol(*pre),
+                lamports_to_sol(*post)
+            )?;
+        }
+    }
+    Ok(())
+}
+
+fn write_log_messages<W: io::Write>(
+    w: &mut W,
+    log_messages: Option<&Vec<String>>,
+    prefix: &str,
+) -> io::Result<()> {
+    if let Some(log_messages) = log_messages {
+        if !log_messages.is_empty() {
+            writeln!(w, "{}Log Messages:", prefix,)?;
+            for log_message in log_messages {
+                writeln!(w, "{}  {}", prefix, log_message)?;
+            }
+        }
+    }
+    Ok(())
+}
+
 pub fn println_transaction(
-    transaction: &Transaction,
-    transaction_status: &Option<UiTransactionStatusMeta>,
+    transaction: &VersionedTransaction,
+    transaction_status: Option<&UiTransactionStatusMeta>,
     prefix: &str,
     sigverify_status: Option<&[CliSignatureVerificationStatus]>,
     block_time: Option<UnixTimestamp>,
@@ -374,6 +607,7 @@ pub fn println_transaction(
         prefix,
         sigverify_status,
         block_time,
+        CliTimezone::Local,
     )
     .is_ok()
     {
@@ -385,23 +619,24 @@ pub fn println_transaction(
 
 pub fn writeln_transaction(
     f: &mut dyn fmt::Write,
-    transaction: &Transaction,
-    transaction_status: &Option<UiTransactionStatusMeta>,
+    transaction: &VersionedTransaction,
+    transaction_status: Option<&UiTransactionStatusMeta>,
     prefix: &str,
     sigverify_status: Option<&[CliSignatureVerificationStatus]>,
     block_time: Option<UnixTimestamp>,
 ) -> fmt::Result {
     let mut w = Vec::new();
-    if write_transaction(
+    let write_result = write_transaction(
         &mut w,
         transaction,
         transaction_status,
         prefix,
         sigverify_status,
         block_time,
-    )
-    .is_ok()
-    {
+        CliTimezone::Local,
+    );
+
+    if write_result.is_ok() {
         if let Ok(s) = String::from_utf8(w) {
             write!(f, "{}", s)?;
         }
@@ -427,7 +662,215 @@ pub fn unix_timestamp_to_string(unix_timestamp: UnixTimestamp) -> String {
 
 #[cfg(test)]
 mod test {
-    use {super::*, solana_sdk::pubkey::Pubkey};
+    use {
+        super::*,
+        solana_sdk::{
+            message::{
+                v0::{self, LoadedAddresses},
+                Message as LegacyMessage, MessageHeader, VersionedMessage,
+            },
+            pubkey::Pubkey,
+            signature::{Keypair, Signer},
+            transaction::Transaction,
+        },
+        solana_transaction_status::{Reward, RewardType, TransactionStatusMeta},
+        std::io::BufWriter,
+    };
+
+    fn new_test_keypair() -> Keypair {
+        let secret = ed25519_dalek::SecretKey::from_bytes(&[0u8; 32]).unwrap();
+        let public = ed25519_dalek::PublicKey::from(&secret);
+        let keypair = ed25519_dalek::Keypair { secret, public };
+        Keypair::from_bytes(&keypair.to_bytes()).unwrap()
+    }
+
+    fn new_test_v0_transaction() -> VersionedTransaction {
+        let keypair = new_test_keypair();
+        let account_key = Pubkey::new_from_array([1u8; 32]);
+        let address_table_key = Pubkey::new_from_array([2u8; 32]);
+        VersionedTransaction::try_new(
+            VersionedMessage::V0(v0::Message {
+                header: MessageHeader {
+                    num_required_signatures: 1,
+                    num_readonly_signed_accounts: 0,
+                    num_readonly_unsigned_accounts: 1,
+                },
+                recent_blockhash: Hash::default(),
+                account_keys: vec![keypair.pubkey(), account_key],
+                address_table_lookups: vec![MessageAddressTableLookup {
+                    account_key: address_table_key,
+                    writable_indexes: vec![0],
+                    readonly_indexes: vec![1],
+                }],
+                instructions: vec![CompiledInstruction::new_from_raw_parts(
+                    3,
+                    vec![],
+                    vec![1, 2],
+                )],
+            }),
+            &[&keypair],
+        )
+        .unwrap()
+    }
+
+    #[test]
+    fn test_write_legacy_transaction() {
+        let keypair = new_test_keypair();
+        let account_key = Pubkey::new_from_array([1u8; 32]);
+        let transaction = VersionedTransaction::from(Transaction::new(
+            &[&keypair],
+            LegacyMessage {
+                header: MessageHeader {
+                    num_required_signatures: 1,
+                    num_readonly_signed_accounts: 0,
+                    num_readonly_unsigned_accounts: 1,
+                },
+                recent_blockhash: Hash::default(),
+                account_keys: vec![keypair.pubkey(), account_key],
+                instructions: vec![CompiledInstruction::new_from_raw_parts(1, vec![], vec![0])],
+            },
+            Hash::default(),
+        ));
+
+        let sigverify_status = CliSignatureVerificationStatus::verify_transaction(&transaction);
+        let meta = TransactionStatusMeta {
+            status: Ok(()),
+            fee: 5000,
+            pre_balances: vec![5000, 10_000],
+            post_balances: vec![0, 9_900],
+            inner_instructions: None,
+            log_messages: Some(vec!["Test message".to_string()]),
+            pre_token_balances: None,
+            post_token_balances: None,
+            rewards: Some(vec![Reward {
+                pubkey: account_key.to_string(),
+                lamports: -100,
+                post_balance: 9_900,
+                reward_type: Some(RewardType::Rent),
+                commission: None,
+            }]),
+            loaded_addresses: LoadedAddresses::default(),
+        };
+
+        let output = {
+            let mut write_buffer = BufWriter::new(Vec::new());
+            write_transaction(
+                &mut write_buffer,
+                &transaction,
+                Some(&meta.into()),
+                "",
+                Some(&sigverify_status),
+                Some(1628633791),
+                CliTimezone::Utc,
+            )
+            .unwrap();
+            let bytes = write_buffer.into_inner().unwrap();
+            String::from_utf8(bytes).unwrap()
+        };
+
+        assert_eq!(
+            output,
+            r#"Block Time: 2021-08-10T22:16:31Z
+Version: legacy
+Recent Blockhash: 11111111111111111111111111111111
+Signature 0: 5pkjrE4VBa3Bu9CMKXgh1U345cT1gGo8QBVRTzHAo6gHeiPae5BTbShP15g6NgqRMNqu8Qrhph1ATmrfC1Ley3rx (pass)
+Account 0: srw- 4zvwRjXUKGfvwnParsHAS3HuSVzV5cA4McphgmoCtajS (fee payer)
+Account 1: -r-x 4vJ9JU1bJJE96FWSJKvHsmmFADCg4gpZQff4P3bkLKi
+Instruction 0
+  Program:   4vJ9JU1bJJE96FWSJKvHsmmFADCg4gpZQff4P3bkLKi (1)
+  Account 0: 4zvwRjXUKGfvwnParsHAS3HuSVzV5cA4McphgmoCtajS (0)
+  Data: []
+Status: Ok
+  Fee: ◎0.000005
+  Account 0 balance: ◎0.000005 -> ◎0
+  Account 1 balance: ◎0.00001 -> ◎0.0000099
+Log Messages:
+  Test message
+Rewards:
+  Address                                            Type        Amount            New Balance         \0
+  4vJ9JU1bJJE96FWSJKvHsmmFADCg4gpZQff4P3bkLKi        rent        -◎0.000000100     ◎0.000009900       \0
+"#.replace("\\0", "") // replace marker used to subvert trailing whitespace linter on CI
+        );
+    }
+
+    #[test]
+    fn test_write_v0_transaction() {
+        let versioned_tx = new_test_v0_transaction();
+        let sigverify_status = CliSignatureVerificationStatus::verify_transaction(&versioned_tx);
+        let address_table_entry1 = Pubkey::new_from_array([3u8; 32]);
+        let address_table_entry2 = Pubkey::new_from_array([4u8; 32]);
+        let loaded_addresses = LoadedAddresses {
+            writable: vec![address_table_entry1],
+            readonly: vec![address_table_entry2],
+        };
+        let meta = TransactionStatusMeta {
+            status: Ok(()),
+            fee: 5000,
+            pre_balances: vec![5000, 10_000, 15_000, 20_000],
+            post_balances: vec![0, 10_000, 14_900, 20_000],
+            inner_instructions: None,
+            log_messages: Some(vec!["Test message".to_string()]),
+            pre_token_balances: None,
+            post_token_balances: None,
+            rewards: Some(vec![Reward {
+                pubkey: address_table_entry1.to_string(),
+                lamports: -100,
+                post_balance: 14_900,
+                reward_type: Some(RewardType::Rent),
+                commission: None,
+            }]),
+            loaded_addresses,
+        };
+
+        let output = {
+            let mut write_buffer = BufWriter::new(Vec::new());
+            write_transaction(
+                &mut write_buffer,
+                &versioned_tx,
+                Some(&meta.into()),
+                "",
+                Some(&sigverify_status),
+                Some(1628633791),
+                CliTimezone::Utc,
+            )
+            .unwrap();
+            let bytes = write_buffer.into_inner().unwrap();
+            String::from_utf8(bytes).unwrap()
+        };
+
+        assert_eq!(
+            output,
+            r#"Block Time: 2021-08-10T22:16:31Z
+Version: 0
+Recent Blockhash: 11111111111111111111111111111111
+Signature 0: 5iEy3TT3ZhTA1NkuCY8GrQGNVY8d5m1bpjdh5FT3Ca4Py81fMipAZjafDuKJKrkw5q5UAAd8oPcgZ4nyXpHt4Fp7 (pass)
+Account 0: srw- 4zvwRjXUKGfvwnParsHAS3HuSVzV5cA4McphgmoCtajS (fee payer)
+Account 1: -r-- 4vJ9JU1bJJE96FWSJKvHsmmFADCg4gpZQff4P3bkLKi
+Account 2: -rw- Unknown Address (uses lookup 0 and index 0)
+Account 3: -r-x Unknown Address (uses lookup 0 and index 1)
+Instruction 0
+  Program:   Unknown Address (uses lookup 0 and index 1) (3)
+  Account 0: 4vJ9JU1bJJE96FWSJKvHsmmFADCg4gpZQff4P3bkLKi (1)
+  Account 1: Unknown Address (uses lookup 0 and index 0) (2)
+  Data: []
+Address Table Lookup 0
+  Table Account: 8qbHbw2BbbTHBW1sbeqakYXVKRQM8Ne7pLK7m6CVfeR
+  Writable Indexes: [0]
+  Readonly Indexes: [1]
+Status: Ok
+  Fee: ◎0.000005
+  Account 0 balance: ◎0.000005 -> ◎0
+  Account 1 balance: ◎0.00001
+  Account 2 balance: ◎0.000015 -> ◎0.0000149
+  Account 3 balance: ◎0.00002
+Log Messages:
+  Test message
+Rewards:
+  Address                                            Type        Amount            New Balance         \0
+  CktRuQ2mttgRGkXJtyksdKHjUdc2C4TgDzyB98oEzy8        rent        -◎0.000000100     ◎0.000014900       \0
+"#.replace("\\0", "") // replace marker used to subvert trailing whitespace linter on CI
+        );
+    }
 
     #[test]
     fn test_format_labeled_address() {

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.10.0"
+version = "1.10.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,43 +13,43 @@ documentation = "https://docs.rs/solana-cli"
 bincode = "1.3.3"
 bs58 = "0.4.0"
 clap = "2.33.1"
-criterion-stats = "0.3.0"
-ctrlc = { version = "3.2.1", features = ["termination"] }
 console = "0.15.0"
 const_format = "0.2.22"
+criterion-stats = "0.3.0"
 crossbeam-channel = "0.5"
-log = "0.4.14"
+ctrlc = { version = "3.2.1", features = ["termination"] }
 humantime = "2.0.1"
+log = "0.4.14"
 num-traits = "0.2"
 pretty-hex = "0.2.1"
-reqwest = { version = "0.11.9", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-semver = "1.0.5"
+reqwest = { version = "0.11.10", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+semver = "1.0.6"
 serde = "1.0.136"
 serde_derive = "1.0.103"
-serde_json = "1.0.78"
-solana-account-decoder = { path = "../account-decoder", version = "=1.10.0" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.10.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.10.0" }
-solana-cli-config = { path = "../cli-config", version = "=1.10.0" }
-solana-cli-output = { path = "../cli-output", version = "=1.10.0" }
-solana-client = { path = "../client", version = "=1.10.0" }
-solana-config-program = { path = "../programs/config", version = "=1.10.0" }
-solana-faucet = { path = "../faucet", version = "=1.10.0" }
-solana-logger = { path = "../logger", version = "=1.10.0" }
-solana-program-runtime = { path = "../program-runtime", version = "=1.10.0" }
+serde_json = "1.0.79"
+solana-account-decoder = { path = "../account-decoder", version = "=1.10.6" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.10.6" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.10.6" }
+solana-cli-config = { path = "../cli-config", version = "=1.10.6" }
+solana-cli-output = { path = "../cli-output", version = "=1.10.6" }
+solana-client = { path = "../client", version = "=1.10.6" }
+solana-config-program = { path = "../programs/config", version = "=1.10.6" }
+solana-faucet = { path = "../faucet", version = "=1.10.6" }
+solana-logger = { path = "../logger", version = "=1.10.6" }
+solana-program-runtime = { path = "../program-runtime", version = "=1.10.6" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.10.6" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.6" }
+solana-version = { path = "../version", version = "=1.10.6" }
+solana-vote-program = { path = "../programs/vote", version = "=1.10.6" }
 solana_rbpf = "=0.2.24"
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
-solana-version = { path = "../version", version = "=1.10.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.10.0" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 thiserror = "1.0.30"
 tiny-bip39 = "0.8.2"
 
 [dev-dependencies]
-solana-streamer = { path = "../streamer", version = "=1.10.0" }
-solana-test-validator = { path = "../test-validator", version = "=1.10.0" }
+solana-streamer = { path = "../streamer", version = "=1.10.6" }
+solana-test-validator = { path = "../test-validator", version = "=1.10.6" }
 tempfile = "3.3.0"
 
 [[bin]]

cli/src/cli.rs

@@ -30,7 +30,7 @@ use {
         pubkey::Pubkey,
         signature::{Signature, Signer, SignerError},
         stake::{instruction::LockupArgs, state::Lockup},
-        transaction::{Transaction, TransactionError},
+        transaction::{TransactionError, VersionedTransaction},
     },
     solana_vote_program::vote_state::VoteAuthorize,
     std::{collections::HashMap, error, io::stdout, str::FromStr, sync::Arc, time::Duration},
@@ -88,6 +88,7 @@ pub enum CliCommand {
         timeout: Duration,
         blockhash: Option<Hash>,
         print_timestamp: bool,
+        additional_fee: Option<u32>,
     },
     Rent {
         data_length: usize,
@@ -161,6 +162,7 @@ pub enum CliCommand {
         address: Option<SignerIndex>,
         use_deprecated_loader: bool,
         allow_excessive_balance: bool,
+        skip_fee_check: bool,
     },
     Program(ProgramCliCommand),
     // Stake Commands
@@ -384,7 +386,7 @@ pub enum CliCommand {
         seed: String,
         program_id: Pubkey,
     },
-    DecodeTransaction(Transaction),
+    DecodeTransaction(VersionedTransaction),
     ResolveSigner(Option<String>),
     ShowAccount {
         pubkey: Pubkey,
@@ -743,6 +745,7 @@ pub fn parse_command(
                 signers.push(signer);
                 1
             });
+            let skip_fee_check = matches.is_present("skip_fee_check");
 
             Ok(CliCommandInfo {
                 command: CliCommand::Deploy {
@@ -750,6 +753,7 @@ pub fn parse_command(
                     address,
                     use_deprecated_loader: matches.is_present("use_deprecated_loader"),
                     allow_excessive_balance: matches.is_present("allow_excessive_balance"),
+                    skip_fee_check,
                 },
                 signers,
             })
@@ -977,6 +981,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             timeout,
             blockhash,
             print_timestamp,
+            additional_fee,
         } => process_ping(
             &rpc_client,
             config,
@@ -985,6 +990,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             timeout,
             blockhash,
             *print_timestamp,
+            additional_fee,
         ),
         CliCommand::Rent {
             data_length,
@@ -1126,6 +1132,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             address,
             use_deprecated_loader,
             allow_excessive_balance,
+            skip_fee_check,
         } => process_deploy(
             rpc_client,
             config,
@@ -1133,6 +1140,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             *address,
             *use_deprecated_loader,
             *allow_excessive_balance,
+            *skip_fee_check,
         ),
         CliCommand::Program(program_subcommand) => {
             process_program_subcommand(rpc_client, config, program_subcommand)
@@ -1964,6 +1972,7 @@ mod tests {
                     address: None,
                     use_deprecated_loader: false,
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 },
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -1986,6 +1995,7 @@ mod tests {
                     address: Some(1),
                     use_deprecated_loader: false,
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 },
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -2379,6 +2389,7 @@ mod tests {
             address: None,
             use_deprecated_loader: false,
             allow_excessive_balance: false,
+            skip_fee_check: false,
         };
         config.output_format = OutputFormat::JsonCompact;
         let result = process_command(&config);
@@ -2399,6 +2410,7 @@ mod tests {
             address: None,
             use_deprecated_loader: false,
             allow_excessive_balance: false,
+            skip_fee_check: false,
         };
         assert!(process_command(&config).is_err());
     }

cli/src/cluster_query.rs

@@ -33,12 +33,14 @@ use {
         rpc_request::DELINQUENT_VALIDATOR_SLOT_DISTANCE,
         rpc_response::SlotInfo,
     },
+    solana_program_runtime::compute_budget,
     solana_remote_wallet::remote_wallet::RemoteWalletManager,
     solana_sdk::{
         account::from_account,
         account_utils::StateMut,
         clock::{self, Clock, Slot},
         commitment_config::CommitmentConfig,
+        compute_budget::ComputeBudgetInstruction,
         epoch_schedule::Epoch,
         hash::Hash,
         message::Message,
@@ -269,6 +271,13 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .default_value("15")
                         .help("Wait up to timeout seconds for transaction confirmation"),
                 )
+                .arg(
+                    Arg::with_name("additional_fee")
+                        .long("additional-fee")
+                        .value_name("NUMBER")
+                        .takes_value(true)
+                        .help("Request additional-fee for transaction"),
+                )
                 .arg(blockhash_arg()),
         )
         .subcommand(
@@ -513,6 +522,7 @@ pub fn parse_cluster_ping(
     let timeout = Duration::from_secs(value_t_or_exit!(matches, "timeout", u64));
     let blockhash = value_of(matches, BLOCKHASH_ARG.name);
     let print_timestamp = matches.is_present("print_timestamp");
+    let additional_fee = value_of(matches, "additional_fee");
     Ok(CliCommandInfo {
         command: CliCommand::Ping {
             interval,
@@ -520,6 +530,7 @@ pub fn parse_cluster_ping(
             timeout,
             blockhash,
             print_timestamp,
+            additional_fee,
         },
         signers: vec![default_signer.signer_from_path(matches, wallet_manager)?],
     })
@@ -1047,6 +1058,7 @@ pub fn process_get_block(
             RpcBlockConfig {
                 encoding: Some(UiTransactionEncoding::Base64),
                 commitment: Some(CommitmentConfig::confirmed()),
+                max_supported_transaction_version: Some(0),
                 ..RpcBlockConfig::default()
             },
         )?
@@ -1350,6 +1362,7 @@ pub fn process_ping(
     timeout: &Duration,
     fixed_blockhash: &Option<Hash>,
     print_timestamp: bool,
+    additional_fee: &Option<u32>,
 ) -> ProcessResult {
     let (signal_sender, signal_receiver) = unbounded();
     ctrlc::set_handler(move || {
@@ -1374,6 +1387,7 @@ pub fn process_ping(
             blockhash_from_cluster = true;
         }
     }
+
     'mainloop: for seq in 0..count.unwrap_or(std::u64::MAX) {
         let now = Instant::now();
         if fixed_blockhash.is_none() && now.duration_since(blockhash_acquired).as_secs() > 60 {
@@ -1388,8 +1402,18 @@ pub fn process_ping(
         lamports += 1;
 
         let build_message = |lamports| {
-            let ix = system_instruction::transfer(&config.signers[0].pubkey(), &to, lamports);
-            Message::new(&[ix], Some(&config.signers[0].pubkey()))
+            let mut ixs = vec![system_instruction::transfer(
+                &config.signers[0].pubkey(),
+                &to,
+                lamports,
+            )];
+            if let Some(additional_fee) = additional_fee {
+                ixs.push(ComputeBudgetInstruction::request_units(
+                    compute_budget::DEFAULT_UNITS,
+                    *additional_fee,
+                ));
+            }
+            Message::new(&ixs, Some(&config.signers[0].pubkey()))
         };
         let (message, _) = resolve_spend_tx_and_check_account_balance(
             rpc_client,
@@ -2019,6 +2043,7 @@ pub fn process_transaction_history(
                     RpcTransactionConfig {
                         encoding: Some(UiTransactionEncoding::Base64),
                         commitment: Some(CommitmentConfig::confirmed()),
+                        max_supported_transaction_version: Some(0),
                     },
                 ) {
                     Ok(confirmed_transaction) => {
@@ -2028,7 +2053,7 @@ pub fn process_transaction_history(
                                 .transaction
                                 .decode()
                                 .expect("Successful decode"),
-                            &confirmed_transaction.transaction.meta,
+                            confirmed_transaction.transaction.meta.as_ref(),
                             "  ",
                             None,
                             None,
@@ -2312,6 +2337,7 @@ mod tests {
                         Hash::from_str("4CCNp28j6AhGq7PkjPDP4wbQWBS8LLbQin2xV5n8frKX").unwrap()
                     ),
                     print_timestamp: true,
+                    additional_fee: None,
                 },
                 signers: vec![default_keypair.into()],
             }

cli/src/program.rs

@@ -66,6 +66,7 @@ pub enum ProgramCliCommand {
         is_final: bool,
         max_len: Option<usize>,
         allow_excessive_balance: bool,
+        skip_fee_check: bool,
     },
     WriteBuffer {
         program_location: String,
@@ -73,6 +74,7 @@ pub enum ProgramCliCommand {
         buffer_pubkey: Option<Pubkey>,
         buffer_authority_signer_index: Option<SignerIndex>,
         max_len: Option<usize>,
+        skip_fee_check: bool,
     },
     SetBufferAuthority {
         buffer_pubkey: Pubkey,
@@ -114,6 +116,13 @@ impl ProgramSubCommands for App<'_, '_> {
             SubCommand::with_name("program")
                 .about("Program management")
                 .setting(AppSettings::SubcommandRequiredElseHelp)
+                .arg(
+                    Arg::with_name("skip_fee_check")
+                        .long("skip-fee-check")
+                        .hidden(true)
+                        .takes_value(false)
+                        .global(true)
+                )
                 .subcommand(
                     SubCommand::with_name("deploy")
                         .about("Deploy a program")
@@ -406,6 +415,12 @@ impl ProgramSubCommands for App<'_, '_> {
                         .long("allow-excessive-deploy-account-balance")
                         .takes_value(false)
                         .help("Use the designated program id, even if the account already holds a large balance of SOL")
+                )
+                .arg(
+                    Arg::with_name("skip_fee_check")
+                        .long("skip-fee-check")
+                        .hidden(true)
+                        .takes_value(false)
                 ),
         )
     }
@@ -416,7 +431,14 @@ pub fn parse_program_subcommand(
     default_signer: &DefaultSigner,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let response = match matches.subcommand() {
+    let (subcommand, sub_matches) = matches.subcommand();
+    let matches_skip_fee_check = matches.is_present("skip_fee_check");
+    let sub_matches_skip_fee_check = sub_matches
+        .map(|m| m.is_present("skip_fee_check"))
+        .unwrap_or(false);
+    let skip_fee_check = matches_skip_fee_check || sub_matches_skip_fee_check;
+
+    let response = match (subcommand, sub_matches) {
         ("deploy", Some(matches)) => {
             let mut bulk_signers = vec![Some(
                 default_signer.signer_from_path(matches, wallet_manager)?,
@@ -476,6 +498,7 @@ pub fn parse_program_subcommand(
                     is_final: matches.is_present("final"),
                     max_len,
                     allow_excessive_balance: matches.is_present("allow_excessive_balance"),
+                    skip_fee_check,
                 }),
                 signers: signer_info.signers,
             }
@@ -521,6 +544,7 @@ pub fn parse_program_subcommand(
                     buffer_authority_signer_index: signer_info
                         .index_of_or_none(buffer_authority_pubkey),
                     max_len,
+                    skip_fee_check,
                 }),
                 signers: signer_info.signers,
             }
@@ -669,6 +693,7 @@ pub fn process_program_subcommand(
             is_final,
             max_len,
             allow_excessive_balance,
+            skip_fee_check,
         } => process_program_deploy(
             rpc_client,
             config,
@@ -681,6 +706,7 @@ pub fn process_program_subcommand(
             *is_final,
             *max_len,
             *allow_excessive_balance,
+            *skip_fee_check,
         ),
         ProgramCliCommand::WriteBuffer {
             program_location,
@@ -688,6 +714,7 @@ pub fn process_program_subcommand(
             buffer_pubkey,
             buffer_authority_signer_index,
             max_len,
+            skip_fee_check,
         } => process_write_buffer(
             rpc_client,
             config,
@@ -696,6 +723,7 @@ pub fn process_program_subcommand(
             *buffer_pubkey,
             *buffer_authority_signer_index,
             *max_len,
+            *skip_fee_check,
         ),
         ProgramCliCommand::SetBufferAuthority {
             buffer_pubkey,
@@ -793,6 +821,7 @@ fn process_program_deploy(
     is_final: bool,
     max_len: Option<usize>,
     allow_excessive_balance: bool,
+    skip_fee_check: bool,
 ) -> ProcessResult {
     let (words, mnemonic, buffer_keypair) = create_ephemeral_keypair()?;
     let (buffer_provided, buffer_signer, buffer_pubkey) = if let Some(i) = buffer_signer_index {
@@ -947,6 +976,7 @@ fn process_program_deploy(
             &buffer_pubkey,
             Some(upgrade_authority_signer),
             allow_excessive_balance,
+            skip_fee_check,
         )
     } else {
         do_process_program_upgrade(
@@ -957,6 +987,7 @@ fn process_program_deploy(
             config.signers[upgrade_authority_signer_index],
             &buffer_pubkey,
             buffer_signer,
+            skip_fee_check,
         )
     };
     if result.is_ok() && is_final {
@@ -983,6 +1014,7 @@ fn process_write_buffer(
     buffer_pubkey: Option<Pubkey>,
     buffer_authority_signer_index: Option<SignerIndex>,
     max_len: Option<usize>,
+    skip_fee_check: bool,
 ) -> ProcessResult {
     // Create ephemeral keypair to use for Buffer account, if not provided
     let (words, mnemonic, buffer_keypair) = create_ephemeral_keypair()?;
@@ -1050,6 +1082,7 @@ fn process_write_buffer(
         &buffer_pubkey,
         Some(buffer_authority),
         true,
+        skip_fee_check,
     );
 
     if result.is_err() && buffer_signer_index.is_none() && buffer_signer.is_some() {
@@ -1636,6 +1669,7 @@ pub fn process_deploy(
     buffer_signer_index: Option<SignerIndex>,
     use_deprecated_loader: bool,
     allow_excessive_balance: bool,
+    skip_fee_check: bool,
 ) -> ProcessResult {
     // Create ephemeral keypair to use for Buffer account, if not provided
     let (words, mnemonic, buffer_keypair) = create_ephemeral_keypair()?;
@@ -1666,6 +1700,7 @@ pub fn process_deploy(
         &buffer_signer.pubkey(),
         Some(buffer_signer),
         allow_excessive_balance,
+        skip_fee_check,
     );
     if result.is_err() && buffer_signer_index.is_none() {
         report_ephemeral_mnemonic(words, mnemonic);
@@ -1704,6 +1739,7 @@ fn do_process_program_write_and_deploy(
     buffer_pubkey: &Pubkey,
     buffer_authority_signer: Option<&dyn Signer>,
     allow_excessive_balance: bool,
+    skip_fee_check: bool,
 ) -> ProcessResult {
     // Build messages to calculate fees
     let mut messages: Vec<&Message> = Vec::new();
@@ -1834,7 +1870,9 @@ fn do_process_program_write_and_deploy(
         messages.push(message);
     }
 
-    check_payer(&rpc_client, config, balance_needed, &messages)?;
+    if !skip_fee_check {
+        check_payer(&rpc_client, config, balance_needed, &messages)?;
+    }
 
     send_deploy_messages(
         rpc_client,
@@ -1868,6 +1906,7 @@ fn do_process_program_upgrade(
     upgrade_authority: &dyn Signer,
     buffer_pubkey: &Pubkey,
     buffer_signer: Option<&dyn Signer>,
+    skip_fee_check: bool,
 ) -> ProcessResult {
     let loader_id = bpf_loader_upgradeable::id();
     let data_len = program_data.len();
@@ -1967,7 +2006,10 @@ fn do_process_program_upgrade(
     );
     messages.push(&final_message);
 
-    check_payer(&rpc_client, config, balance_needed, &messages)?;
+    if !skip_fee_check {
+        check_payer(&rpc_client, config, balance_needed, &messages)?;
+    }
+
     send_deploy_messages(
         rpc_client,
         config,
@@ -2255,6 +2297,7 @@ mod tests {
                     is_final: false,
                     max_len: None,
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 }),
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -2281,6 +2324,7 @@ mod tests {
                     is_final: false,
                     max_len: Some(42),
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 }),
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -2309,6 +2353,7 @@ mod tests {
                     is_final: false,
                     max_len: None,
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 }),
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -2339,6 +2384,7 @@ mod tests {
                     is_final: false,
                     max_len: None,
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 }),
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -2368,6 +2414,7 @@ mod tests {
                     is_final: false,
                     max_len: None,
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 }),
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -2400,6 +2447,7 @@ mod tests {
                     is_final: false,
                     max_len: None,
                     allow_excessive_balance: false,
+                    skip_fee_check: false,
                 }),
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -2427,6 +2475,7 @@ mod tests {
                     upgrade_authority_signer_index: 0,
                     is_final: true,
                     max_len: None,
+                    skip_fee_check: false,
                     allow_excessive_balance: false,
                 }),
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
@@ -2460,6 +2509,7 @@ mod tests {
                     buffer_pubkey: None,
                     buffer_authority_signer_index: Some(0),
                     max_len: None,
+                    skip_fee_check: false,
                 }),
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -2483,6 +2533,7 @@ mod tests {
                     buffer_pubkey: None,
                     buffer_authority_signer_index: Some(0),
                     max_len: Some(42),
+                    skip_fee_check: false,
                 }),
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -2509,6 +2560,7 @@ mod tests {
                     buffer_pubkey: Some(buffer_keypair.pubkey()),
                     buffer_authority_signer_index: Some(0),
                     max_len: None,
+                    skip_fee_check: false,
                 }),
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -2538,6 +2590,7 @@ mod tests {
                     buffer_pubkey: None,
                     buffer_authority_signer_index: Some(1),
                     max_len: None,
+                    skip_fee_check: false,
                 }),
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -2572,6 +2625,7 @@ mod tests {
                     buffer_pubkey: Some(buffer_keypair.pubkey()),
                     buffer_authority_signer_index: Some(2),
                     max_len: None,
+                    skip_fee_check: false,
                 }),
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -3014,6 +3068,7 @@ mod tests {
                 is_final: false,
                 max_len: None,
                 allow_excessive_balance: false,
+                skip_fee_check: false,
             }),
             signers: vec![&default_keypair],
             output_format: OutputFormat::JsonCompact,

cli/src/stake.rs

@@ -1383,17 +1383,12 @@ pub fn process_stake_authorize(
             };
             if let Some(authorized) = authorized {
                 match authorization_type {
-                    StakeAuthorize::Staker => {
-                        // first check authorized withdrawer
-                        check_current_authority(&authorized.withdrawer, &authority.pubkey())
-                            .or_else(|_| {
-                                // ...then check authorized staker. If neither matches, error will
-                                // print the stake key as `expected`
-                                check_current_authority(&authorized.staker, &authority.pubkey())
-                            })?;
-                    }
+                    StakeAuthorize::Staker => check_current_authority(
+                        &[authorized.withdrawer, authorized.staker],
+                        &authority.pubkey(),
+                    )?,
                     StakeAuthorize::Withdrawer => {
-                        check_current_authority(&authorized.withdrawer, &authority.pubkey())?;
+                        check_current_authority(&[authorized.withdrawer], &authority.pubkey())?;
                     }
                 }
             } else {
@@ -1935,7 +1930,7 @@ pub fn process_stake_set_lockup(
         };
         if let Some(lockup) = lockup {
             if lockup.custodian != Pubkey::default() {
-                check_current_authority(&lockup.custodian, &custodian.pubkey())?;
+                check_current_authority(&[lockup.custodian], &custodian.pubkey())?;
             }
         } else {
             return Err(CliError::RpcRequestError(format!(
@@ -2119,13 +2114,13 @@ fn get_stake_account_state(
 }
 
 pub(crate) fn check_current_authority(
-    account_current_authority: &Pubkey,
+    permitted_authorities: &[Pubkey],
     provided_current_authority: &Pubkey,
 ) -> Result<(), CliError> {
-    if account_current_authority != provided_current_authority {
+    if !permitted_authorities.contains(provided_current_authority) {
         Err(CliError::RpcRequestError(format!(
-            "Invalid current authority provided: {:?}, expected {:?}",
-            provided_current_authority, account_current_authority
+            "Invalid authority provided: {:?}, expected {:?}",
+            provided_current_authority, permitted_authorities
         )))
     } else {
         Ok(())

cli/src/validator_info.rs

@@ -291,8 +291,12 @@ pub fn process_set_validator_info(
     // Check existence of validator-info account
     let balance = rpc_client.get_balance(&info_pubkey).unwrap_or(0);
 
-    let lamports =
-        rpc_client.get_minimum_balance_for_rent_exemption(ValidatorInfo::max_space() as usize)?;
+    let keys = vec![
+        (validator_info::id(), false),
+        (config.signers[0].pubkey(), true),
+    ];
+    let data_len = ValidatorInfo::max_space() + ConfigKeys::serialized_size(keys.clone());
+    let lamports = rpc_client.get_minimum_balance_for_rent_exemption(data_len as usize)?;
 
     let signers = if balance == 0 {
         if info_pubkey != info_keypair.pubkey() {
@@ -308,10 +312,7 @@ pub fn process_set_validator_info(
     };
 
     let build_message = |lamports| {
-        let keys = vec![
-            (validator_info::id(), false),
-            (config.signers[0].pubkey(), true),
-        ];
+        let keys = keys.clone();
         if balance == 0 {
             println!(
                 "Publishing info for Validator {:?}",

cli/src/vote.rs

@@ -910,7 +910,10 @@ pub fn process_vote_authorize(
                             "Invalid vote account state; no authorized voters found".to_string(),
                         )
                     })?;
-                check_current_authority(&current_authorized_voter, &authorized.pubkey())?;
+                check_current_authority(
+                    &[current_authorized_voter, vote_state.authorized_withdrawer],
+                    &authorized.pubkey(),
+                )?;
                 if let Some(signer) = new_authorized_signer {
                     if signer.is_interactive() {
                         return Err(CliError::BadParameter(format!(
@@ -927,7 +930,7 @@ pub fn process_vote_authorize(
                 (new_authorized_pubkey, "new_authorized_pubkey".to_string()),
             )?;
             if let Some(vote_state) = vote_state {
-                check_current_authority(&vote_state.authorized_withdrawer, &authorized.pubkey())?
+                check_current_authority(&[vote_state.authorized_withdrawer], &authorized.pubkey())?
             }
         }
     }

cli/src/wallet.rs

@@ -37,9 +37,12 @@ use {
         stake,
         system_instruction::{self, SystemError},
         system_program,
-        transaction::Transaction,
+        transaction::{Transaction, VersionedTransaction},
+    },
+    solana_transaction_status::{
+        EncodableWithMeta, EncodedConfirmedTransactionWithStatusMeta, EncodedTransaction,
+        TransactionBinaryEncoding, UiTransactionEncoding,
     },
-    solana_transaction_status::{Encodable, EncodedTransaction, UiTransactionEncoding},
     std::{fmt::Write as FmtWrite, fs::File, io::Write, sync::Arc},
 };
 
@@ -189,7 +192,7 @@ impl WalletSubCommands for App<'_, '_> {
                     Arg::with_name("encoding")
                         .index(2)
                         .value_name("ENCODING")
-                        .possible_values(&["base58", "base64"]) // Subset of `UiTransactionEncoding` enum
+                        .possible_values(&["base58", "base64"]) // Variants of `TransactionBinaryEncoding` enum
                         .default_value("base58")
                         .takes_value(true)
                         .required(true)
@@ -341,13 +344,13 @@ pub fn parse_balance(
 
 pub fn parse_decode_transaction(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let blob = value_t_or_exit!(matches, "transaction", String);
-    let encoding = match matches.value_of("encoding").unwrap() {
-        "base58" => UiTransactionEncoding::Base58,
-        "base64" => UiTransactionEncoding::Base64,
+    let binary_encoding = match matches.value_of("encoding").unwrap() {
+        "base58" => TransactionBinaryEncoding::Base58,
+        "base64" => TransactionBinaryEncoding::Base64,
         _ => unreachable!(),
     };
 
-    let encoded_transaction = EncodedTransaction::Binary(blob, encoding);
+    let encoded_transaction = EncodedTransaction::Binary(blob, binary_encoding);
     if let Some(transaction) = encoded_transaction.decode() {
         Ok(CliCommandInfo {
             command: CliCommand::DecodeTransaction(transaction),
@@ -559,22 +562,25 @@ pub fn process_confirm(
                         RpcTransactionConfig {
                             encoding: Some(UiTransactionEncoding::Base64),
                             commitment: Some(CommitmentConfig::confirmed()),
+                            max_supported_transaction_version: Some(0),
                         },
                     ) {
                         Ok(confirmed_transaction) => {
-                            let decoded_transaction = confirmed_transaction
-                                .transaction
-                                .transaction
-                                .decode()
-                                .expect("Successful decode");
-                            let json_transaction =
-                                decoded_transaction.encode(UiTransactionEncoding::Json);
+                            let EncodedConfirmedTransactionWithStatusMeta {
+                                block_time,
+                                slot,
+                                transaction: transaction_with_meta,
+                            } = confirmed_transaction;
+
+                            let decoded_transaction =
+                                transaction_with_meta.transaction.decode().unwrap();
+                            let json_transaction = decoded_transaction.json_encode();
 
                             transaction = Some(CliTransaction {
                                 transaction: json_transaction,
-                                meta: confirmed_transaction.transaction.meta,
-                                block_time: confirmed_transaction.block_time,
-                                slot: Some(confirmed_transaction.slot),
+                                meta: transaction_with_meta.meta,
+                                block_time,
+                                slot: Some(slot),
                                 decoded_transaction,
                                 prefix: "  ".to_string(),
                                 sigverify_status: vec![],
@@ -606,11 +612,14 @@ pub fn process_confirm(
 }
 
 #[allow(clippy::unnecessary_wraps)]
-pub fn process_decode_transaction(config: &CliConfig, transaction: &Transaction) -> ProcessResult {
+pub fn process_decode_transaction(
+    config: &CliConfig,
+    transaction: &VersionedTransaction,
+) -> ProcessResult {
     let sigverify_status = CliSignatureVerificationStatus::verify_transaction(transaction);
     let decode_transaction = CliTransaction {
         decoded_transaction: transaction.clone(),
-        transaction: transaction.encode(UiTransactionEncoding::Json),
+        transaction: transaction.json_encode(),
         meta: None,
         block_time: None,
         slot: None,

cli/tests/program.rs

@@ -62,6 +62,7 @@ fn test_cli_program_deploy_non_upgradeable() {
         address: None,
         use_deprecated_loader: false,
         allow_excessive_balance: false,
+        skip_fee_check: false,
     };
     config.output_format = OutputFormat::JsonCompact;
     let response = process_command(&config);
@@ -91,6 +92,7 @@ fn test_cli_program_deploy_non_upgradeable() {
         address: Some(1),
         use_deprecated_loader: false,
         allow_excessive_balance: false,
+        skip_fee_check: false,
     };
     process_command(&config).unwrap();
     let account1 = rpc_client
@@ -118,6 +120,7 @@ fn test_cli_program_deploy_non_upgradeable() {
         address: Some(1),
         use_deprecated_loader: false,
         allow_excessive_balance: false,
+        skip_fee_check: false,
     };
     process_command(&config).unwrap_err();
 
@@ -127,6 +130,7 @@ fn test_cli_program_deploy_non_upgradeable() {
         address: Some(1),
         use_deprecated_loader: false,
         allow_excessive_balance: true,
+        skip_fee_check: false,
     };
     process_command(&config).unwrap();
     let account2 = rpc_client
@@ -193,6 +197,7 @@ fn test_cli_program_deploy_no_authority() {
         upgrade_authority_signer_index: 1,
         is_final: true,
         max_len: None,
+        skip_fee_check: false,
     });
     config.output_format = OutputFormat::JsonCompact;
     let response = process_command(&config);
@@ -218,6 +223,7 @@ fn test_cli_program_deploy_no_authority() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap_err();
 }
@@ -278,6 +284,7 @@ fn test_cli_program_deploy_with_authority() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: Some(max_len),
+        skip_fee_check: false,
     });
     config.output_format = OutputFormat::JsonCompact;
     let response = process_command(&config);
@@ -325,6 +332,7 @@ fn test_cli_program_deploy_with_authority() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: Some(max_len),
+        skip_fee_check: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -366,6 +374,7 @@ fn test_cli_program_deploy_with_authority() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: Some(max_len),
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
     let program_account = rpc_client.get_account(&program_pubkey).unwrap();
@@ -420,6 +429,7 @@ fn test_cli_program_deploy_with_authority() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
     let program_account = rpc_client.get_account(&program_pubkey).unwrap();
@@ -494,6 +504,7 @@ fn test_cli_program_deploy_with_authority() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap_err();
 
@@ -509,6 +520,7 @@ fn test_cli_program_deploy_with_authority() {
         upgrade_authority_signer_index: 1,
         is_final: true,
         max_len: None,
+        skip_fee_check: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -611,6 +623,7 @@ fn test_cli_program_close_program() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: Some(max_len),
+        skip_fee_check: false,
     });
     config.output_format = OutputFormat::JsonCompact;
     process_command(&config).unwrap();
@@ -695,6 +708,7 @@ fn test_cli_program_write_buffer() {
         buffer_pubkey: None,
         buffer_authority_signer_index: None,
         max_len: None,
+        skip_fee_check: false,
     });
     config.output_format = OutputFormat::JsonCompact;
     let response = process_command(&config);
@@ -729,6 +743,7 @@ fn test_cli_program_write_buffer() {
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: None,
         max_len: Some(max_len),
+        skip_fee_check: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -790,6 +805,7 @@ fn test_cli_program_write_buffer() {
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: Some(2),
         max_len: None,
+        skip_fee_check: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -827,6 +843,7 @@ fn test_cli_program_write_buffer() {
         buffer_pubkey: None,
         buffer_authority_signer_index: Some(2),
         max_len: None,
+        skip_fee_check: false,
     });
     let response = process_command(&config);
     let json: Value = serde_json::from_str(&response.unwrap()).unwrap();
@@ -899,6 +916,7 @@ fn test_cli_program_write_buffer() {
         buffer_pubkey: None,
         buffer_authority_signer_index: None,
         max_len: None,
+        skip_fee_check: false,
     });
     config.output_format = OutputFormat::JsonCompact;
     let response = process_command(&config);
@@ -938,6 +956,7 @@ fn test_cli_program_write_buffer() {
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: None,
         max_len: None, //Some(max_len),
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
     config.signers = vec![&keypair, &buffer_keypair];
@@ -951,6 +970,7 @@ fn test_cli_program_write_buffer() {
         upgrade_authority_signer_index: 1,
         is_final: true,
         max_len: None,
+        skip_fee_check: false,
     });
     config.output_format = OutputFormat::JsonCompact;
     let error = process_command(&config).unwrap_err();
@@ -1008,6 +1028,7 @@ fn test_cli_program_set_buffer_authority() {
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: None,
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
     let buffer_account = rpc_client.get_account(&buffer_keypair.pubkey()).unwrap();
@@ -1123,6 +1144,7 @@ fn test_cli_program_mismatch_buffer_authority() {
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: Some(2),
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
     let buffer_account = rpc_client.get_account(&buffer_keypair.pubkey()).unwrap();
@@ -1145,6 +1167,7 @@ fn test_cli_program_mismatch_buffer_authority() {
         upgrade_authority_signer_index: 1,
         is_final: true,
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap_err();
 
@@ -1160,6 +1183,7 @@ fn test_cli_program_mismatch_buffer_authority() {
         upgrade_authority_signer_index: 1,
         is_final: true,
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
 }
@@ -1216,6 +1240,7 @@ fn test_cli_program_show() {
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: Some(2),
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
 
@@ -1275,6 +1300,7 @@ fn test_cli_program_show() {
         upgrade_authority_signer_index: 1,
         is_final: false,
         max_len: Some(max_len),
+        skip_fee_check: false,
     });
     config.output_format = OutputFormat::JsonCompact;
     let min_slot = rpc_client.get_slot().unwrap();
@@ -1401,6 +1427,7 @@ fn test_cli_program_dump() {
         buffer_pubkey: Some(buffer_keypair.pubkey()),
         buffer_authority_signer_index: Some(2),
         max_len: None,
+        skip_fee_check: false,
     });
     process_command(&config).unwrap();
 

client-test/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client-test"
-version = "1.10.0"
+version = "1.10.6"
 description = "Solana RPC Test"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -8,30 +8,31 @@ license = "Apache-2.0"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-client-test"
 edition = "2021"
+publish = false
 
 [dependencies]
-futures-util = "0.3.19"
-serde_json = "1.0.78"
-serial_test = "0.5.1"
-solana-client = { path = "../client", version = "=1.10.0" }
-solana-ledger = { path = "../ledger", version = "=1.10.0" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-solana-merkle-tree = { path = "../merkle-tree", version = "=1.10.0" }
-solana-metrics = { path = "../metrics", version = "=1.10.0" }
-solana-perf = { path = "../perf", version = "=1.10.0" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.10.0" }
-solana-rpc = { path = "../rpc", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-streamer = { path = "../streamer", version = "=1.10.0" }
-solana-test-validator = { path = "../test-validator", version = "=1.10.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
-solana-version = { path = "../version", version = "=1.10.0" }
+futures-util = "0.3.21"
+serde_json = "1.0.79"
+serial_test = "0.6.0"
+solana-client = { path = "../client", version = "=1.10.6" }
+solana-ledger = { path = "../ledger", version = "=1.10.6" }
+solana-measure = { path = "../measure", version = "=1.10.6" }
+solana-merkle-tree = { path = "../merkle-tree", version = "=1.10.6" }
+solana-metrics = { path = "../metrics", version = "=1.10.6" }
+solana-perf = { path = "../perf", version = "=1.10.6" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.10.6" }
+solana-rpc = { path = "../rpc", version = "=1.10.6" }
+solana-runtime = { path = "../runtime", version = "=1.10.6" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
+solana-streamer = { path = "../streamer", version = "=1.10.6" }
+solana-test-validator = { path = "../test-validator", version = "=1.10.6" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.6" }
+solana-version = { path = "../version", version = "=1.10.6" }
 systemstat = "0.1.10"
 tokio = { version = "1", features = ["full"] }
 
 [dev-dependencies]
-solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-logger = { path = "../logger", version = "=1.10.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client-test/tests/client.rs

@@ -15,7 +15,7 @@ use {
     solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path},
     solana_rpc::{
         optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank,
-        rpc::create_test_transactions_and_populate_blockstore,
+        rpc::{create_test_transaction_entries, populate_blockstore_for_tests},
         rpc_pubsub_service::{PubSubConfig, PubSubService},
         rpc_subscriptions::RpcSubscriptions,
     },
@@ -36,7 +36,9 @@ use {
     },
     solana_streamer::socket::SocketAddrSpace,
     solana_test_validator::TestValidator,
-    solana_transaction_status::{ConfirmedBlock, TransactionDetails, UiTransactionEncoding},
+    solana_transaction_status::{
+        BlockEncodingOptions, ConfirmedBlock, TransactionDetails, UiTransactionEncoding,
+    },
     std::{
         collections::HashSet,
         net::{IpAddr, SocketAddr},
@@ -230,9 +232,12 @@ fn test_block_subscription() {
     let max_complete_transaction_status_slot = Arc::new(AtomicU64::new(blockstore.max_root()));
     bank.transfer(rent_exempt_amount, &alice, &keypair2.pubkey())
         .unwrap();
-    let _confirmed_block_signatures = create_test_transactions_and_populate_blockstore(
-        vec![&alice, &keypair1, &keypair2, &keypair3],
-        0,
+    populate_blockstore_for_tests(
+        create_test_transaction_entries(
+            vec![&alice, &keypair1, &keypair2, &keypair3],
+            bank.clone(),
+        )
+        .0,
         bank,
         blockstore.clone(),
         max_complete_transaction_status_slot,
@@ -270,6 +275,7 @@ fn test_block_subscription() {
             encoding: Some(UiTransactionEncoding::Json),
             transaction_details: Some(TransactionDetails::Signatures),
             show_rewards: None,
+            max_supported_transaction_version: None,
         }),
     )
     .unwrap();
@@ -281,14 +287,17 @@ fn test_block_subscription() {
     match maybe_actual {
         Ok(actual) => {
             let versioned_block = blockstore.get_complete_block(slot, false).unwrap();
-            let legacy_block = ConfirmedBlock::from(versioned_block)
-                .into_legacy_block()
+            let confirmed_block = ConfirmedBlock::from(versioned_block);
+            let block = confirmed_block
+                .encode_with_options(
+                    UiTransactionEncoding::Json,
+                    BlockEncodingOptions {
+                        transaction_details: TransactionDetails::Signatures,
+                        show_rewards: false,
+                        max_supported_transaction_version: None,
+                    },
+                )
                 .unwrap();
-            let block = legacy_block.configure(
-                UiTransactionEncoding::Json,
-                TransactionDetails::Signatures,
-                false,
-            );
             assert_eq!(actual.value.slot, slot);
             assert!(block.eq(&actual.value.block.unwrap()));
         }

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.10.0"
+version = "1.10.6"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,31 +10,41 @@ license = "Apache-2.0"
 edition = "2021"
 
 [dependencies]
+async-mutex = "1.4.0"
 async-trait = "0.1.52"
 base64 = "0.13.0"
 bincode = "1.3.3"
 bs58 = "0.4.0"
+bytes = "1.1.0"
 clap = "2.33.0"
 crossbeam-channel = "0.5"
-futures-util = "0.3.19"
+futures = "0.3"
+futures-util = "0.3.21"
 indicatif = "0.16.2"
+itertools = "0.10.2"
 jsonrpc-core = "18.0.0"
+lazy_static = "1.4.0"
 log = "0.4.14"
+quinn = "0.8.0"
+rand = "0.7.0"
+rand_chacha = "0.2.2"
 rayon = "1.5.1"
-reqwest = { version = "0.11.9", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-semver = "1.0.5"
+reqwest = { version = "0.11.10", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+rustls = { version = "0.20.2", features = ["dangerous_configuration"] }
+semver = "1.0.6"
 serde = "1.0.136"
 serde_derive = "1.0.103"
-serde_json = "1.0.78"
-solana-account-decoder = { path = "../account-decoder", version = "=1.10.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.10.0" }
-solana-faucet = { path = "../faucet", version = "=1.10.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
-solana-version = { path = "../version", version = "=1.10.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.10.0" }
+serde_json = "1.0.79"
+solana-account-decoder = { path = "../account-decoder", version = "=1.10.6" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.10.6" }
+solana-faucet = { path = "../faucet", version = "=1.10.6" }
+solana-measure = { path = "../measure", version = "=1.10.6" }
+solana-net-utils = { path = "../net-utils", version = "=1.10.6" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
+solana-streamer = { path = "../streamer", version = "=1.10.6" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.6" }
+solana-version = { path = "../version", version = "=1.10.6" }
+solana-vote-program = { path = "../programs/vote", version = "=1.10.6" }
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 tokio-stream = "0.1.8"
@@ -45,7 +55,7 @@ url = "2.2.2"
 [dev-dependencies]
 assert_matches = "1.5.0"
 jsonrpc-http-server = "18.0.0"
-solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-logger = { path = "../logger", version = "=1.10.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/client_error.rs

@@ -1,6 +1,7 @@
 pub use reqwest;
 use {
     crate::{rpc_request, rpc_response},
+    quinn::{ConnectError, WriteError},
     solana_faucet::faucet::FaucetError,
     solana_sdk::{
         signature::SignerError, transaction::TransactionError, transport::TransportError,
@@ -72,6 +73,18 @@ impl From<ClientErrorKind> for TransportError {
     }
 }
 
+impl From<WriteError> for ClientErrorKind {
+    fn from(write_error: WriteError) -> Self {
+        Self::Custom(format!("{:?}", write_error))
+    }
+}
+
+impl From<ConnectError> for ClientErrorKind {
+    fn from(connect_error: ConnectError) -> Self {
+        Self::Custom(format!("{:?}", connect_error))
+    }
+}
+
 #[derive(Error, Debug)]
 #[error("{kind}")]
 pub struct ClientError {

client/src/connection_cache.rs

@@ -0,0 +1,241 @@
+use {
+    crate::{
+        quic_client::QuicTpuConnection, tpu_connection::TpuConnection, udp_client::UdpTpuConnection,
+    },
+    lazy_static::lazy_static,
+    solana_sdk::{transaction::VersionedTransaction, transport::TransportError},
+    std::{
+        collections::{hash_map::Entry, BTreeMap, HashMap},
+        net::{SocketAddr, UdpSocket},
+        sync::{Arc, Mutex},
+    },
+};
+
+// Should be non-zero
+static MAX_CONNECTIONS: usize = 64;
+
+#[derive(Clone)]
+enum Connection {
+    Udp(Arc<UdpTpuConnection>),
+    Quic(Arc<QuicTpuConnection>),
+}
+
+struct ConnMap {
+    // Keeps track of the connection associated with an addr and the last time it was used
+    map: HashMap<SocketAddr, (Connection, u64)>,
+    // Helps to find the least recently used connection. The search and inserts are O(log(n))
+    // but since we're bounding the size of the collections, this should be constant
+    // (and hopefully negligible) time. In theory, we can do this in constant time
+    // with a queue implemented as a doubly-linked list (and all the
+    // HashMap entries holding a "pointer" to the corresponding linked-list node),
+    // so we can push, pop and bump a used connection back to the end of the queue in O(1) time, but
+    // that seems non-"Rust-y" and low bang/buck. This is still pretty terrible though...
+    last_used_times: BTreeMap<u64, SocketAddr>,
+    ticks: u64,
+    use_quic: bool,
+}
+
+impl ConnMap {
+    pub fn new() -> Self {
+        Self {
+            map: HashMap::new(),
+            last_used_times: BTreeMap::new(),
+            ticks: 0,
+            use_quic: false,
+        }
+    }
+
+    pub fn set_use_quic(&mut self, use_quic: bool) {
+        self.use_quic = use_quic;
+    }
+}
+
+lazy_static! {
+    static ref CONNECTION_MAP: Mutex<ConnMap> = Mutex::new(ConnMap::new());
+}
+
+pub fn set_use_quic(use_quic: bool) {
+    let mut map = (*CONNECTION_MAP).lock().unwrap();
+    map.set_use_quic(use_quic);
+}
+
+#[allow(dead_code)]
+// TODO: see https://github.com/solana-labs/solana/issues/23661
+// remove lazy_static and optimize and refactor this
+fn get_connection(addr: &SocketAddr) -> Connection {
+    let mut map = (*CONNECTION_MAP).lock().unwrap();
+    let ticks = map.ticks;
+    let use_quic = map.use_quic;
+    let (conn, target_ticks) = match map.map.entry(*addr) {
+        Entry::Occupied(mut entry) => {
+            let mut pair = entry.get_mut();
+            let old_ticks = pair.1;
+            pair.1 = ticks;
+            (pair.0.clone(), old_ticks)
+        }
+        Entry::Vacant(entry) => {
+            let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+            // TODO: see https://github.com/solana-labs/solana/issues/23659
+            // make it configurable (e.g. via the command line) whether to use UDP or Quic
+
+            let conn = if use_quic {
+                Connection::Quic(Arc::new(QuicTpuConnection::new(send_socket, *addr)))
+            } else {
+                Connection::Udp(Arc::new(UdpTpuConnection::new(send_socket, *addr)))
+            };
+
+            entry.insert((conn.clone(), ticks));
+            (conn, ticks)
+        }
+    };
+
+    let num_connections = map.map.len();
+    if num_connections > MAX_CONNECTIONS {
+        let (old_ticks, target_addr) = {
+            let (old_ticks, target_addr) = map.last_used_times.iter().next().unwrap();
+            (*old_ticks, *target_addr)
+        };
+        map.map.remove(&target_addr);
+        map.last_used_times.remove(&old_ticks);
+    }
+
+    if target_ticks != ticks {
+        map.last_used_times.remove(&target_ticks);
+    }
+    map.last_used_times.insert(ticks, *addr);
+
+    map.ticks += 1;
+    conn
+}
+
+// TODO: see https://github.com/solana-labs/solana/issues/23851
+// use enum_dispatch and get rid of this tedious code.
+// The main blocker to using enum_dispatch right now is that
+// the it doesn't work with static methods like TpuConnection::new
+// which is used by thin_client. This will be eliminated soon
+// once thin_client is moved to using this connection cache.
+// Once that is done, we will migrate to using enum_dispatch
+// This will be done in a followup to
+// https://github.com/solana-labs/solana/pull/23817
+pub fn send_wire_transaction_batch(
+    packets: &[&[u8]],
+    addr: &SocketAddr,
+) -> Result<(), TransportError> {
+    let conn = get_connection(addr);
+    match conn {
+        Connection::Udp(conn) => conn.send_wire_transaction_batch(packets),
+        Connection::Quic(conn) => conn.send_wire_transaction_batch(packets),
+    }
+}
+
+pub fn send_wire_transaction(
+    wire_transaction: &[u8],
+    addr: &SocketAddr,
+) -> Result<(), TransportError> {
+    let conn = get_connection(addr);
+    match conn {
+        Connection::Udp(conn) => conn.send_wire_transaction(wire_transaction),
+        Connection::Quic(conn) => conn.send_wire_transaction(wire_transaction),
+    }
+}
+
+pub fn serialize_and_send_transaction(
+    transaction: &VersionedTransaction,
+    addr: &SocketAddr,
+) -> Result<(), TransportError> {
+    let conn = get_connection(addr);
+    match conn {
+        Connection::Udp(conn) => conn.serialize_and_send_transaction(transaction),
+        Connection::Quic(conn) => conn.serialize_and_send_transaction(transaction),
+    }
+}
+
+pub fn par_serialize_and_send_transaction_batch(
+    transactions: &[VersionedTransaction],
+    addr: &SocketAddr,
+) -> Result<(), TransportError> {
+    let conn = get_connection(addr);
+    match conn {
+        Connection::Udp(conn) => conn.par_serialize_and_send_transaction_batch(transactions),
+        Connection::Quic(conn) => conn.par_serialize_and_send_transaction_batch(transactions),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use {
+        crate::{
+            connection_cache::{get_connection, Connection, CONNECTION_MAP, MAX_CONNECTIONS},
+            tpu_connection::TpuConnection,
+        },
+        rand::{Rng, SeedableRng},
+        rand_chacha::ChaChaRng,
+        std::net::{IpAddr, SocketAddr},
+    };
+
+    fn get_addr(rng: &mut ChaChaRng) -> SocketAddr {
+        let a = rng.gen_range(1, 255);
+        let b = rng.gen_range(1, 255);
+        let c = rng.gen_range(1, 255);
+        let d = rng.gen_range(1, 255);
+
+        let addr_str = format!("{}.{}.{}.{}:80", a, b, c, d);
+
+        addr_str.parse().expect("Invalid address")
+    }
+
+    fn ip(conn: Connection) -> IpAddr {
+        match conn {
+            Connection::Udp(conn) => conn.tpu_addr().ip(),
+            Connection::Quic(conn) => conn.tpu_addr().ip(),
+        }
+    }
+
+    #[test]
+    fn test_connection_cache() {
+        // Allow the test to run deterministically
+        // with the same pseudorandom sequence between runs
+        // and on different platforms - the cryptographic security
+        // property isn't important here but ChaChaRng provides a way
+        // to get the same pseudorandom sequence on different platforms
+        let mut rng = ChaChaRng::seed_from_u64(42);
+
+        // Generate a bunch of random addresses and create TPUConnections to them
+        // Since TPUConnection::new is infallible, it should't matter whether or not
+        // we can actually connect to those addresses - TPUConnection implementations should either
+        // be lazy and not connect until first use or handle connection errors somehow
+        // (without crashing, as would be required in a real practical validator)
+        let first_addr = get_addr(&mut rng);
+        assert!(ip(get_connection(&first_addr)) == first_addr.ip());
+        let addrs = (0..MAX_CONNECTIONS)
+            .into_iter()
+            .map(|_| {
+                let addr = get_addr(&mut rng);
+                get_connection(&addr);
+                addr
+            })
+            .collect::<Vec<_>>();
+        {
+            let map = (*CONNECTION_MAP).lock().unwrap();
+            addrs.iter().for_each(|a| {
+                let conn = map.map.get(a).expect("Address not found");
+                assert!(a.ip() == ip(conn.0.clone()));
+            });
+
+            assert!(map.map.get(&first_addr).is_none());
+        }
+
+        // Test that get_connection updates which connection is next up for eviction
+        // when an existing connection is used. Initially, addrs[0] should be next up for eviction, since
+        // it was the earliest added. But we do get_connection(&addrs[0]), thereby using
+        // that connection, and bumping it back to the end of the queue. So addrs[1] should be
+        // the next up for eviction. So we add a new connection, and test that addrs[0] is not
+        // evicted but addrs[1] is.
+        get_connection(&addrs[0]);
+        get_connection(&get_addr(&mut rng));
+
+        let map = (*CONNECTION_MAP).lock().unwrap();
+        assert!(map.map.get(&addrs[0]).is_some());
+        assert!(map.map.get(&addrs[1]).is_none());
+    }
+}

client/src/lib.rs

@@ -4,12 +4,14 @@ extern crate serde_derive;
 
 pub mod blockhash_query;
 pub mod client_error;
+pub mod connection_cache;
 pub(crate) mod http_sender;
 pub(crate) mod mock_sender;
 pub mod nonblocking;
 pub mod nonce_utils;
 pub mod perf_utils;
 pub mod pubsub_client;
+pub mod quic_client;
 pub mod rpc_cache;
 pub mod rpc_client;
 pub mod rpc_config;
@@ -18,11 +20,13 @@ pub mod rpc_deprecated_config;
 pub mod rpc_filter;
 pub mod rpc_request;
 pub mod rpc_response;
-pub(crate) mod rpc_sender;
+pub mod rpc_sender;
 pub mod spinner;
 pub mod thin_client;
 pub mod tpu_client;
+pub mod tpu_connection;
 pub mod transaction_executor;
+pub mod udp_client;
 
 pub mod mock_sender_for_cli {
     /// Magic `SIGNATURE` value used by `solana-cli` unit tests.

client/src/mock_sender.rs

@@ -28,13 +28,13 @@ use {
         pubkey::Pubkey,
         signature::Signature,
         sysvar::epoch_schedule::EpochSchedule,
-        transaction::{self, Transaction, TransactionError},
+        transaction::{self, Transaction, TransactionError, TransactionVersion},
     },
     solana_transaction_status::{
         EncodedConfirmedBlock, EncodedConfirmedTransactionWithStatusMeta, EncodedTransaction,
-        EncodedTransactionWithStatusMeta, Rewards, TransactionConfirmationStatus,
-        TransactionStatus, UiCompiledInstruction, UiMessage, UiRawMessage, UiTransaction,
-        UiTransactionEncoding, UiTransactionStatusMeta,
+        EncodedTransactionWithStatusMeta, Rewards, TransactionBinaryEncoding,
+        TransactionConfirmationStatus, TransactionStatus, UiCompiledInstruction, UiMessage,
+        UiRawMessage, UiTransaction, UiTransactionStatusMeta,
     },
     solana_version::Version,
     std::{collections::HashMap, net::SocketAddr, str::FromStr, sync::RwLock},
@@ -192,6 +192,7 @@ impl RpcSender for MockSender {
             "getTransaction" => serde_json::to_value(EncodedConfirmedTransactionWithStatusMeta {
                 slot: 2,
                 transaction: EncodedTransactionWithStatusMeta {
+                    version: Some(TransactionVersion::LEGACY),
                     transaction: EncodedTransaction::Json(
                         UiTransaction {
                             signatures: vec!["3AsdoALgZFuq2oUVWrDYhg2pNeaLJKPLf8hU2mQ6U8qJxeJ6hsrPVpMn9ma39DtfYCrDQSvngWRP8NnTpEhezJpE".to_string()],
@@ -213,6 +214,7 @@ impl RpcSender for MockSender {
                                         accounts: vec![0, 1],
                                         data: "3Bxs49DitAvXtoDR".to_string(),
                                     }],
+                                    address_table_lookups: None,
                                 })
                         }),
                     meta: Some(UiTransactionStatusMeta {
@@ -226,6 +228,7 @@ impl RpcSender for MockSender {
                             pre_token_balances: None,
                             post_token_balances: None,
                             rewards: None,
+                            loaded_addresses: None,
                         }),
                 },
                 block_time: Some(1628633791),
@@ -378,9 +381,10 @@ impl RpcSender for MockSender {
                                  pLHxcaShD81xBNaFDgnA2nkkdHnKtZt4hVSfKAmw3VRZbjrZ7L2fKZBx21CwsG\
                                  hD6onjM2M3qZW5C8J6d1pj41MxKmZgPBSha3MyKkNLkAGFASK"
                             .to_string(),
-                        UiTransactionEncoding::Base58,
+                        TransactionBinaryEncoding::Base58,
                     ),
                     meta: None,
+                    version: Some(TransactionVersion::LEGACY),
                 }],
                 rewards: Rewards::new(),
                 block_time: None,

client/src/nonblocking/rpc_client.rs

@@ -6,6 +6,7 @@
 //!
 //! [JSON-RPC]: https://www.jsonrpc.org/specification
 
+pub use crate::mock_sender::Mocks;
 #[allow(deprecated)]
 use crate::rpc_deprecated_config::{
     RpcConfirmedBlockConfig, RpcConfirmedTransactionConfig,
@@ -15,7 +16,7 @@ use {
     crate::{
         client_error::{ClientError, ClientErrorKind, Result as ClientResult},
         http_sender::HttpSender,
-        mock_sender::{MockSender, Mocks},
+        mock_sender::MockSender,
         rpc_client::{GetConfirmedSignaturesForAddress2Config, RpcClientConfig},
         rpc_config::{RpcAccountInfoConfig, *},
         rpc_request::{RpcError, RpcRequest, RpcResponseErrorData, TokenAccountsFilter},
@@ -146,9 +147,9 @@ impl RpcClient {
     ///
     /// This is the basic constructor, allowing construction with any type of
     /// `RpcSender`. Most applications should use one of the other constructors,
-    /// such as [`new`] and [`new_mock`], which create an `RpcClient`
-    /// encapsulating an [`HttpSender`] and [`MockSender`] respectively.
-    pub(crate) fn new_sender<T: RpcSender + Send + Sync + 'static>(
+    /// such as [`RpcClient::new`], [`RpcClient::new_with_commitment`] or
+    /// [`RpcClient::new_with_timeout`].
+    pub fn new_sender<T: RpcSender + Send + Sync + 'static>(
         sender: T,
         config: RpcClientConfig,
     ) -> Self {
@@ -314,8 +315,34 @@ impl RpcClient {
 
     /// Create a mock `RpcClient`.
     ///
-    /// See the [`MockSender`] documentation for an explanation of
-    /// how it treats the `url` argument.
+    /// A mock `RpcClient` contains an implementation of [`RpcSender`] that does
+    /// not use the network, and instead returns synthetic responses, for use in
+    /// tests.
+    ///
+    /// It is primarily for internal use, with limited customizability, and
+    /// behaviors determined by internal Solana test cases. New users should
+    /// consider implementing `RpcSender` themselves and constructing
+    /// `RpcClient` with [`RpcClient::new_sender`] to get mock behavior.
+    ///
+    /// Unless directed otherwise, a mock `RpcClient` will generally return a
+    /// reasonable default response to any request, at least for [`RpcRequest`]
+    /// values for which responses have been implemented.
+    ///
+    /// This mock can be customized by changing the `url` argument, which is not
+    /// actually a URL, but a simple string directive that changes the mock
+    /// behavior in specific scenarios:
+    ///
+    /// - It is customary to set the `url` to "succeeds" for mocks that should
+    ///   return sucessfully, though this value is not actually interpreted.
+    ///
+    /// - If `url` is "fails" then any call to `send` will return `Ok(Value::Null)`.
+    ///
+    /// - Other possible values of `url` are specific to different `RpcRequest`
+    ///   values. Read the implementation of (non-public) `MockSender` for
+    ///   details.
+    ///
+    /// The [`RpcClient::new_mock_with_mocks`] function offers further
+    /// customization options.
     ///
     /// # Examples
     ///
@@ -341,8 +368,43 @@ impl RpcClient {
 
     /// Create a mock `RpcClient`.
     ///
-    /// See the [`MockSender`] documentation for an explanation of how it treats
-    /// the `url` argument.
+    /// A mock `RpcClient` contains an implementation of [`RpcSender`] that does
+    /// not use the network, and instead returns synthetic responses, for use in
+    /// tests.
+    ///
+    /// It is primarily for internal use, with limited customizability, and
+    /// behaviors determined by internal Solana test cases. New users should
+    /// consider implementing `RpcSender` themselves and constructing
+    /// `RpcClient` with [`RpcClient::new_sender`] to get mock behavior.
+    ///
+    /// Unless directed otherwise, a mock `RpcClient` will generally return a
+    /// reasonable default response to any request, at least for [`RpcRequest`]
+    /// values for which responses have been implemented.
+    ///
+    /// This mock can be customized in two ways:
+    ///
+    /// 1) By changing the `url` argument, which is not actually a URL, but a
+    ///    simple string directive that changes the mock behavior in specific
+    ///    scenarios.
+    ///
+    ///    It is customary to set the `url` to "succeeds" for mocks that should
+    ///    return sucessfully, though this value is not actually interpreted.
+    ///
+    ///    If `url` is "fails" then any call to `send` will return `Ok(Value::Null)`.
+    ///
+    ///    Other possible values of `url` are specific to different `RpcRequest`
+    ///    values. Read the implementation of `MockSender` (which is non-public)
+    ///    for details.
+    ///
+    /// 2) Custom responses can be configured by providing [`Mocks`]. This type
+    ///    is a [`HashMap`] from [`RpcRequest`] to a JSON [`Value`] response,
+    ///    Any entries in this map override the default behavior for the given
+    ///    request.
+    ///
+    /// The [`RpcClient::new_mock_with_mocks`] function offers further
+    /// customization options.
+    ///
+    /// [`HashMap`]: std::collections::HashMap
     ///
     /// # Examples
     ///
@@ -2411,6 +2473,7 @@ impl RpcClient {
     ///     transaction_details: Some(TransactionDetails::None),
     ///     rewards: Some(true),
     ///     commitment: None,
+    ///     max_supported_transaction_version: Some(0),
     /// };
     /// let block = rpc_client.get_block_with_config(
     ///     slot,
@@ -3051,6 +3114,7 @@ impl RpcClient {
     /// let config = RpcTransactionConfig {
     ///     encoding: Some(UiTransactionEncoding::Json),
     ///     commitment: Some(CommitmentConfig::confirmed()),
+    ///     max_supported_transaction_version: Some(0),
     /// };
     /// let transaction = rpc_client.get_transaction_with_config(
     ///     &signature,

client/src/quic_client.rs

@@ -0,0 +1,213 @@
+//! Simple client that connects to a given UDP port with the QUIC protocol and provides
+//! an interface for sending transactions which is restricted by the server's flow control.
+
+use {
+    crate::{client_error::ClientErrorKind, tpu_connection::TpuConnection},
+    async_mutex::Mutex,
+    futures::future::join_all,
+    itertools::Itertools,
+    quinn::{ClientConfig, Endpoint, EndpointConfig, NewConnection, WriteError},
+    solana_sdk::{
+        quic::{QUIC_MAX_CONCURRENT_STREAMS, QUIC_PORT_OFFSET},
+        transport::Result as TransportResult,
+    },
+    std::{
+        net::{SocketAddr, UdpSocket},
+        sync::Arc,
+    },
+    tokio::runtime::Runtime,
+};
+
+struct SkipServerVerification;
+
+impl SkipServerVerification {
+    pub fn new() -> Arc<Self> {
+        Arc::new(Self)
+    }
+}
+
+impl rustls::client::ServerCertVerifier for SkipServerVerification {
+    fn verify_server_cert(
+        &self,
+        _end_entity: &rustls::Certificate,
+        _intermediates: &[rustls::Certificate],
+        _server_name: &rustls::ServerName,
+        _scts: &mut dyn Iterator<Item = &[u8]>,
+        _ocsp_response: &[u8],
+        _now: std::time::SystemTime,
+    ) -> Result<rustls::client::ServerCertVerified, rustls::Error> {
+        Ok(rustls::client::ServerCertVerified::assertion())
+    }
+}
+
+struct QuicClient {
+    runtime: Runtime,
+    endpoint: Endpoint,
+    connection: Arc<Mutex<Option<Arc<NewConnection>>>>,
+    addr: SocketAddr,
+}
+
+pub struct QuicTpuConnection {
+    client: Arc<QuicClient>,
+}
+
+impl TpuConnection for QuicTpuConnection {
+    fn new(client_socket: UdpSocket, tpu_addr: SocketAddr) -> Self {
+        let tpu_addr = SocketAddr::new(tpu_addr.ip(), tpu_addr.port() + QUIC_PORT_OFFSET);
+        let client = Arc::new(QuicClient::new(client_socket, tpu_addr));
+
+        Self { client }
+    }
+
+    fn tpu_addr(&self) -> &SocketAddr {
+        &self.client.addr
+    }
+
+    fn send_wire_transaction<T>(&self, wire_transaction: T) -> TransportResult<()>
+    where
+        T: AsRef<[u8]>,
+    {
+        let _guard = self.client.runtime.enter();
+        let send_buffer = self.client.send_buffer(wire_transaction);
+        self.client.runtime.block_on(send_buffer)?;
+        Ok(())
+    }
+
+    fn send_wire_transaction_batch<T>(&self, buffers: &[T]) -> TransportResult<()>
+    where
+        T: AsRef<[u8]>,
+    {
+        let _guard = self.client.runtime.enter();
+        let send_batch = self.client.send_batch(buffers);
+        self.client.runtime.block_on(send_batch)?;
+        Ok(())
+    }
+}
+
+impl QuicClient {
+    pub fn new(client_socket: UdpSocket, addr: SocketAddr) -> Self {
+        let runtime = tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+
+        let _guard = runtime.enter();
+
+        let crypto = rustls::ClientConfig::builder()
+            .with_safe_defaults()
+            .with_custom_certificate_verifier(SkipServerVerification::new())
+            .with_no_client_auth();
+
+        let create_endpoint = QuicClient::create_endpoint(EndpointConfig::default(), client_socket);
+
+        let mut endpoint = runtime.block_on(create_endpoint);
+
+        endpoint.set_default_client_config(ClientConfig::new(Arc::new(crypto)));
+
+        Self {
+            runtime,
+            endpoint,
+            connection: Arc::new(Mutex::new(None)),
+            addr,
+        }
+    }
+
+    // If this function becomes public, it should be changed to
+    // not expose details of the specific Quic implementation we're using
+    async fn create_endpoint(config: EndpointConfig, client_socket: UdpSocket) -> Endpoint {
+        quinn::Endpoint::new(config, None, client_socket).unwrap().0
+    }
+
+    async fn _send_buffer_using_conn(
+        data: &[u8],
+        connection: &NewConnection,
+    ) -> Result<(), WriteError> {
+        let mut send_stream = connection.connection.open_uni().await?;
+        send_stream.write_all(data).await?;
+        send_stream.finish().await?;
+        Ok(())
+    }
+
+    // Attempts to send data, connecting/reconnecting as necessary
+    // On success, returns the connection used to successfully send the data
+    async fn _send_buffer(&self, data: &[u8]) -> Result<Arc<NewConnection>, WriteError> {
+        let connection = {
+            let mut conn_guard = self.connection.lock().await;
+
+            let maybe_conn = (*conn_guard).clone();
+            match maybe_conn {
+                Some(conn) => conn.clone(),
+                None => {
+                    let connecting = self.endpoint.connect(self.addr, "connect").unwrap();
+                    let connection = Arc::new(connecting.await?);
+                    *conn_guard = Some(connection.clone());
+                    connection
+                }
+            }
+        };
+        match Self::_send_buffer_using_conn(data, &connection).await {
+            Ok(()) => Ok(connection),
+            _ => {
+                let connection = {
+                    let connecting = self.endpoint.connect(self.addr, "connect").unwrap();
+                    let connection = Arc::new(connecting.await?);
+                    let mut conn_guard = self.connection.lock().await;
+                    *conn_guard = Some(connection.clone());
+                    connection
+                };
+                Self::_send_buffer_using_conn(data, &connection).await?;
+                Ok(connection)
+            }
+        }
+    }
+
+    pub async fn send_buffer<T>(&self, data: T) -> Result<(), ClientErrorKind>
+    where
+        T: AsRef<[u8]>,
+    {
+        self._send_buffer(data.as_ref()).await?;
+        Ok(())
+    }
+
+    pub async fn send_batch<T>(&self, buffers: &[T]) -> Result<(), ClientErrorKind>
+    where
+        T: AsRef<[u8]>,
+    {
+        // Start off by "testing" the connection by sending the first transaction
+        // This will also connect to the server if not already connected
+        // and reconnect and retry if the first send attempt failed
+        // (for example due to a timed out connection), returning an error
+        // or the connection that was used to successfully send the transaction.
+        // We will use the returned connection to send the rest of the transactions in the batch
+        // to avoid touching the mutex in self, and not bother reconnecting if we fail along the way
+        // since testing even in the ideal GCE environment has found no cases
+        // where reconnecting and retrying in the middle of a batch send
+        // (i.e. we encounter a connection error in the middle of a batch send, which presumably cannot
+        // be due to a timed out connection) has succeeded
+        if buffers.is_empty() {
+            return Ok(());
+        }
+        let connection = self._send_buffer(buffers[0].as_ref()).await?;
+
+        // Used to avoid dereferencing the Arc multiple times below
+        // by just getting a reference to the NewConnection once
+        let connection_ref: &NewConnection = &connection;
+
+        let chunks = buffers[1..buffers.len()]
+            .iter()
+            .chunks(QUIC_MAX_CONCURRENT_STREAMS);
+
+        let futures = chunks.into_iter().map(|buffs| {
+            join_all(
+                buffs
+                    .into_iter()
+                    .map(|buf| Self::_send_buffer_using_conn(buf.as_ref(), connection_ref)),
+            )
+        });
+
+        for f in futures {
+            f.await.into_iter().try_for_each(|res| res)?;
+        }
+        Ok(())
+    }
+}

client/src/rpc_client.rs

@@ -6,13 +6,14 @@
 //!
 //! [JSON-RPC]: https://www.jsonrpc.org/specification
 
+pub use crate::mock_sender::Mocks;
 #[allow(deprecated)]
 use crate::rpc_deprecated_config::{RpcConfirmedBlockConfig, RpcConfirmedTransactionConfig};
 use {
     crate::{
         client_error::Result as ClientResult,
         http_sender::HttpSender,
-        mock_sender::{MockSender, Mocks},
+        mock_sender::MockSender,
         nonblocking::{self, rpc_client::get_rpc_request_str},
         rpc_config::{RpcAccountInfoConfig, *},
         rpc_request::{RpcRequest, TokenAccountsFilter},
@@ -103,8 +104,8 @@ pub struct GetConfirmedSignaturesForAddress2Config {
 /// [`Processed`] commitment level. These exceptions are noted in the method
 /// documentation.
 ///
-/// [`Finalized`]: CommitmentLevel::Finalized
-/// [`Processed`]: CommitmentLevel::Processed
+/// [`Finalized`]: solana_sdk::commitment_config::CommitmentLevel::Finalized
+/// [`Processed`]: solana_sdk::commitment_config::CommitmentLevel::Processed
 /// [jsonprot]: https://docs.solana.com/developing/clients/jsonrpc-api
 /// [JSON-RPC]: https://www.jsonrpc.org/specification
 /// [slots]: https://docs.solana.com/terminology#slot
@@ -145,6 +146,10 @@ pub struct GetConfirmedSignaturesForAddress2Config {
 /// [`is_timeout`](crate::client_error::reqwest::Error::is_timeout) method
 /// returns `true`. The default timeout is 30 seconds, and may be changed by
 /// calling an appropriate constructor with a `timeout` parameter.
+///
+/// [`ClientError`]: crate::client_error::ClientError
+/// [`ClientErrorKind`]: crate::client_error::ClientErrorKind
+/// [`ClientErrorKind::Reqwest`]: crate::client_error::ClientErrorKind::Reqwest
 pub struct RpcClient {
     rpc_client: nonblocking::rpc_client::RpcClient,
     runtime: Option<tokio::runtime::Runtime>,
@@ -161,9 +166,9 @@ impl RpcClient {
     ///
     /// This is the basic constructor, allowing construction with any type of
     /// `RpcSender`. Most applications should use one of the other constructors,
-    /// such as [`new`] and [`new_mock`], which create an `RpcClient`
-    /// encapsulating an [`HttpSender`] and [`MockSender`] respectively.
-    fn new_sender<T: RpcSender + Send + Sync + 'static>(
+    /// such as [`RpcClient::new`], [`RpcClient::new_with_commitment`] or
+    /// [`RpcClient::new_with_timeout`].
+    pub fn new_sender<T: RpcSender + Send + Sync + 'static>(
         sender: T,
         config: RpcClientConfig,
     ) -> Self {
@@ -186,9 +191,10 @@ impl RpcClient {
     /// "http://localhost:8899".
     ///
     /// The client has a default timeout of 30 seconds, and a default [commitment
-    /// level][cl] of [`Finalized`](CommitmentLevel::Finalized).
+    /// level][cl] of [`Finalized`].
     ///
     /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
+    /// [`Finalized`]: solana_sdk::commitment_config::CommitmentLevel::Finalized
     ///
     /// # Examples
     ///
@@ -211,6 +217,8 @@ impl RpcClient {
     /// The client has a default timeout of 30 seconds, and a user-specified
     /// [`CommitmentLevel`] via [`CommitmentConfig`].
     ///
+    /// [`CommitmentLevel`]: solana_sdk::commitment_config::CommitmentLevel
+    ///
     /// # Examples
     ///
     /// ```
@@ -233,9 +241,10 @@ impl RpcClient {
     /// "http://localhost:8899".
     ///
     /// The client has and a default [commitment level][cl] of
-    /// [`Finalized`](CommitmentLevel::Finalized).
+    /// [`Finalized`].
     ///
     /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
+    /// [`Finalized`]: solana_sdk::commitment_config::CommitmentLevel::Finalized
     ///
     /// # Examples
     ///
@@ -335,8 +344,34 @@ impl RpcClient {
 
     /// Create a mock `RpcClient`.
     ///
-    /// See the [`MockSender`] documentation for an explanation of
-    /// how it treats the `url` argument.
+    /// A mock `RpcClient` contains an implementation of [`RpcSender`] that does
+    /// not use the network, and instead returns synthetic responses, for use in
+    /// tests.
+    ///
+    /// It is primarily for internal use, with limited customizability, and
+    /// behaviors determined by internal Solana test cases. New users should
+    /// consider implementing `RpcSender` themselves and constructing
+    /// `RpcClient` with [`RpcClient::new_sender`] to get mock behavior.
+    ///
+    /// Unless directed otherwise, a mock `RpcClient` will generally return a
+    /// reasonable default response to any request, at least for [`RpcRequest`]
+    /// values for which responses have been implemented.
+    ///
+    /// This mock can be customized by changing the `url` argument, which is not
+    /// actually a URL, but a simple string directive that changes the mock
+    /// behavior in specific scenarios:
+    ///
+    /// - It is customary to set the `url` to "succeeds" for mocks that should
+    ///   return sucessfully, though this value is not actually interpreted.
+    ///
+    /// - If `url` is "fails" then any call to `send` will return `Ok(Value::Null)`.
+    ///
+    /// - Other possible values of `url` are specific to different `RpcRequest`
+    ///   values. Read the implementation of (non-public) `MockSender` for
+    ///   details.
+    ///
+    /// The [`RpcClient::new_mock_with_mocks`] function offers further
+    /// customization options.
     ///
     /// # Examples
     ///
@@ -362,8 +397,43 @@ impl RpcClient {
 
     /// Create a mock `RpcClient`.
     ///
-    /// See the [`MockSender`] documentation for an explanation of how it treats
-    /// the `url` argument.
+    /// A mock `RpcClient` contains an implementation of [`RpcSender`] that does
+    /// not use the network, and instead returns synthetic responses, for use in
+    /// tests.
+    ///
+    /// It is primarily for internal use, with limited customizability, and
+    /// behaviors determined by internal Solana test cases. New users should
+    /// consider implementing `RpcSender` themselves and constructing
+    /// `RpcClient` with [`RpcClient::new_sender`] to get mock behavior.
+    ///
+    /// Unless directed otherwise, a mock `RpcClient` will generally return a
+    /// reasonable default response to any request, at least for [`RpcRequest`]
+    /// values for which responses have been implemented.
+    ///
+    /// This mock can be customized in two ways:
+    ///
+    /// 1) By changing the `url` argument, which is not actually a URL, but a
+    ///    simple string directive that changes the mock behavior in specific
+    ///    scenarios.
+    ///
+    ///    It is customary to set the `url` to "succeeds" for mocks that should
+    ///    return sucessfully, though this value is not actually interpreted.
+    ///
+    ///    If `url` is "fails" then any call to `send` will return `Ok(Value::Null)`.
+    ///
+    ///    Other possible values of `url` are specific to different `RpcRequest`
+    ///    values. Read the implementation of `MockSender` (which is non-public)
+    ///    for details.
+    ///
+    /// 2) Custom responses can be configured by providing [`Mocks`]. This type
+    ///    is a [`HashMap`] from [`RpcRequest`] to a JSON [`Value`] response,
+    ///    Any entries in this map override the default behavior for the given
+    ///    request.
+    ///
+    /// The [`RpcClient::new_mock_with_mocks`] function offers further
+    /// customization options.
+    ///
+    /// [`HashMap`]: std::collections::HashMap
     ///
     /// # Examples
     ///
@@ -397,9 +467,10 @@ impl RpcClient {
     /// Create an HTTP `RpcClient` from a [`SocketAddr`].
     ///
     /// The client has a default timeout of 30 seconds, and a default [commitment
-    /// level][cl] of [`Finalized`](CommitmentLevel::Finalized).
+    /// level][cl] of [`Finalized`].
     ///
     /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
+    /// [`Finalized`]: solana_sdk::commitment_config::CommitmentLevel::Finalized
     ///
     /// # Examples
     ///
@@ -420,6 +491,8 @@ impl RpcClient {
     /// The client has a default timeout of 30 seconds, and a user-specified
     /// [`CommitmentLevel`] via [`CommitmentConfig`].
     ///
+    /// [`CommitmentLevel`]: solana_sdk::commitment_config::CommitmentLevel
+    ///
     /// # Examples
     ///
     /// ```
@@ -442,9 +515,10 @@ impl RpcClient {
 
     /// Create an HTTP `RpcClient` from a [`SocketAddr`] with specified timeout.
     ///
-    /// The client has a default [commitment level][cl] of [`Finalized`](CommitmentLevel::Finalized).
+    /// The client has a default [commitment level][cl] of [`Finalized`].
     ///
     /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
+    /// [`Finalized`]: solana_sdk::commitment_config::CommitmentLevel::Finalized
     ///
     /// # Examples
     ///
@@ -469,7 +543,9 @@ impl RpcClient {
     /// determines how thoroughly committed a transaction must be when waiting
     /// for its confirmation or otherwise checking for confirmation. If not
     /// specified, the default commitment level is
-    /// [`Finalized`](CommitmentLevel::Finalized).
+    /// [`Finalized`].
+    ///
+    /// [`Finalized`]: solana_sdk::commitment_config::CommitmentLevel::Finalized
     ///
     /// The default commitment level is overridden when calling methods that
     /// explicitly provide a [`CommitmentConfig`], like
@@ -503,7 +579,8 @@ impl RpcClient {
     /// containing an [`RpcResponseError`] with `code` set to
     /// [`JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY`].
     ///
-    /// [`RpcResponseError`]: RpcError::RpcResponseError
+    /// [`RpcError`]: crate::rpc_request::RpcError
+    /// [`RpcResponseError`]: crate::rpc_request::RpcError::RpcResponseError
     /// [`JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE
     /// [`JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE
     /// [`JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY
@@ -616,7 +693,8 @@ impl RpcClient {
     /// containing an [`RpcResponseError`] with `code` set to
     /// [`JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY`].
     ///
-    /// [`RpcResponseError`]: RpcError::RpcResponseError
+    /// [`RpcError`]: crate::rpc_request::RpcError
+    /// [`RpcResponseError`]: crate::rpc_request::RpcError::RpcResponseError
     /// [`JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE
     /// [`JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE
     /// [`JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY
@@ -690,7 +768,8 @@ impl RpcClient {
     /// containing an [`RpcResponseError`] with `code` set to
     /// [`JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY`].
     ///
-    /// [`RpcResponseError`]: RpcError::RpcResponseError
+    /// [`RpcError`]: crate::rpc_request::RpcError
+    /// [`RpcResponseError`]: crate::rpc_request::RpcError::RpcResponseError
     /// [`JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE
     /// [`JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_SEND_TRANSACTION_PREFLIGHT_FAILURE
     /// [`JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY`]: crate::rpc_custom_error::JSON_RPC_SERVER_ERROR_NODE_UNHEALTHY
@@ -2034,6 +2113,7 @@ impl RpcClient {
     ///     transaction_details: Some(TransactionDetails::None),
     ///     rewards: Some(true),
     ///     commitment: None,
+    ///     max_supported_transaction_version: Some(0),
     /// };
     /// let block = rpc_client.get_block_with_config(
     ///     slot,
@@ -2101,7 +2181,7 @@ impl RpcClient {
     ///
     /// This method uses the [`Finalized`] [commitment level][cl].
     ///
-    /// [`Finalized`]: CommitmentLevel::Finalized
+    /// [`Finalized`]: solana_sdk::commitment_config::CommitmentLevel::Finalized
     /// [`get_blocks_with_limit`]: RpcClient::get_blocks_with_limit.
     /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
     ///
@@ -2160,7 +2240,7 @@ impl RpcClient {
     /// This method returns an error if the given commitment level is below
     /// [`Confirmed`].
     ///
-    /// [`Confirmed`]: CommitmentLevel::Confirmed
+    /// [`Confirmed`]: solana_sdk::commitment_config::CommitmentLevel::Confirmed
     ///
     /// # RPC Reference
     ///
@@ -2253,7 +2333,7 @@ impl RpcClient {
     /// [`Confirmed`].
     ///
     /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
-    /// [`Confirmed`]: CommitmentLevel::Confirmed
+    /// [`Confirmed`]: solana_sdk::commitment_config::CommitmentLevel::Confirmed
     ///
     /// # RPC Reference
     ///
@@ -2414,7 +2494,7 @@ impl RpcClient {
     /// [`Confirmed`].
     ///
     /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
-    /// [`Confirmed`]: CommitmentLevel::Confirmed
+    /// [`Confirmed`]: solana_sdk::commitment_config::CommitmentLevel::Confirmed
     ///
     /// # RPC Reference
     ///
@@ -2504,7 +2584,7 @@ impl RpcClient {
     ///
     /// This method uses the [`Finalized`] [commitment level][cl].
     ///
-    /// [`Finalized`]: CommitmentLevel::Finalized
+    /// [`Finalized`]: solana_sdk::commitment_config::CommitmentLevel::Finalized
     /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
     ///
     /// # RPC Reference
@@ -2559,7 +2639,7 @@ impl RpcClient {
     /// [`Confirmed`].
     ///
     /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
-    /// [`Confirmed`]: CommitmentLevel::Confirmed
+    /// [`Confirmed`]: solana_sdk::commitment_config::CommitmentLevel::Confirmed
     ///
     /// # RPC Reference
     ///
@@ -2596,6 +2676,7 @@ impl RpcClient {
     /// let config = RpcTransactionConfig {
     ///     encoding: Some(UiTransactionEncoding::Json),
     ///     commitment: Some(CommitmentConfig::confirmed()),
+    ///     max_supported_transaction_version: Some(0),
     /// };
     /// let transaction = rpc_client.get_transaction_with_config(
     ///     &signature,
@@ -2924,7 +3005,7 @@ impl RpcClient {
     ///
     /// This method uses the [`Finalized`] [commitment level][cl].
     ///
-    /// [`Finalized`]: CommitmentLevel::Finalized
+    /// [`Finalized`]: solana_sdk::commitment_config::CommitmentLevel::Finalized
     /// [cl]: https://docs.solana.com/developing/clients/jsonrpc-api#configuring-state-commitment
     ///
     /// # RPC Reference
@@ -3082,6 +3163,7 @@ impl RpcClient {
     /// [`RpcError::ForUser`]. This is unlike [`get_account_with_commitment`],
     /// which returns `Ok(None)` if the account does not exist.
     ///
+    /// [`RpcError::ForUser`]: crate::rpc_request::RpcError::ForUser
     /// [`get_account_with_commitment`]: RpcClient::get_account_with_commitment
     ///
     /// # RPC Reference

client/src/rpc_config.rs

@@ -197,6 +197,7 @@ pub struct RpcBlockSubscribeConfig {
     pub encoding: Option<UiTransactionEncoding>,
     pub transaction_details: Option<TransactionDetails>,
     pub show_rewards: Option<bool>,
+    pub max_supported_transaction_version: Option<u8>,
 }
 
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
@@ -248,6 +249,7 @@ pub struct RpcBlockConfig {
     pub rewards: Option<bool>,
     #[serde(flatten)]
     pub commitment: Option<CommitmentConfig>,
+    pub max_supported_transaction_version: Option<u8>,
 }
 
 impl EncodingConfig for RpcBlockConfig {
@@ -288,6 +290,7 @@ pub struct RpcTransactionConfig {
     pub encoding: Option<UiTransactionEncoding>,
     #[serde(flatten)]
     pub commitment: Option<CommitmentConfig>,
+    pub max_supported_transaction_version: Option<u8>,
 }
 
 impl EncodingConfig for RpcTransactionConfig {

client/src/rpc_custom_error.rs

@@ -3,6 +3,7 @@ use {
     crate::rpc_response::RpcSimulateTransactionResult,
     jsonrpc_core::{Error, ErrorCode},
     solana_sdk::clock::Slot,
+    solana_transaction_status::EncodeError,
     thiserror::Error,
 };
 
@@ -59,7 +60,7 @@ pub enum RpcCustomError {
     #[error("BlockStatusNotAvailableYet")]
     BlockStatusNotAvailableYet { slot: Slot },
     #[error("UnsupportedTransactionVersion")]
-    UnsupportedTransactionVersion,
+    UnsupportedTransactionVersion(u8),
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -68,6 +69,16 @@ pub struct NodeUnhealthyErrorData {
     pub num_slots_behind: Option<Slot>,
 }
 
+impl From<EncodeError> for RpcCustomError {
+    fn from(err: EncodeError) -> Self {
+        match err {
+            EncodeError::UnsupportedTransactionVersion(version) => {
+                Self::UnsupportedTransactionVersion(version)
+            }
+        }
+    }
+}
+
 impl From<RpcCustomError> for Error {
     fn from(e: RpcCustomError) -> Self {
         match e {
@@ -172,9 +183,9 @@ impl From<RpcCustomError> for Error {
                 message: format!("Block status not yet available for slot {}", slot),
                 data: None,
             },
-            RpcCustomError::UnsupportedTransactionVersion => Self {
+            RpcCustomError::UnsupportedTransactionVersion(version) => Self {
                 code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_UNSUPPORTED_TRANSACTION_VERSION),
-                message: "Versioned transactions are not supported".to_string(),
+                message: format!("Transaction version ({}) is not supported", version),
                 data: None,
             },
         }

client/src/rpc_deprecated_config.rs

@@ -71,6 +71,7 @@ impl From<RpcConfirmedBlockConfig> for RpcBlockConfig {
             transaction_details: config.transaction_details,
             rewards: config.rewards,
             commitment: config.commitment,
+            max_supported_transaction_version: None,
         }
     }
 }
@@ -98,6 +99,7 @@ impl From<RpcConfirmedTransactionConfig> for RpcTransactionConfig {
         Self {
             encoding: config.encoding,
             commitment: config.commitment,
+            max_supported_transaction_version: None,
         }
     }
 }

client/src/rpc_response.rs

@@ -117,7 +117,7 @@ pub struct RpcInflationRate {
     pub epoch: Epoch,
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug)]
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcKeyedAccount {
     pub pubkey: String,
@@ -246,7 +246,7 @@ pub struct RpcBlockProductionRange {
     pub last_slot: Slot,
 }
 
-#[derive(Serialize, Deserialize, Clone)]
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcBlockProduction {
     /// Map of leader base58 identity pubkeys to a tuple of `(number of leader slots, number of blocks produced)`
@@ -363,7 +363,7 @@ pub struct RpcAccountBalance {
     pub lamports: u64,
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug)]
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcSupply {
     pub total: u64,
@@ -432,8 +432,8 @@ pub enum RpcBlockUpdateError {
     #[error("block store error")]
     BlockStoreError,
 
-    #[error("unsupported transaction version")]
-    UnsupportedTransactionVersion,
+    #[error("unsupported transaction version ({0})")]
+    UnsupportedTransactionVersion(u8),
 }
 
 #[derive(Serialize, Deserialize, Debug)]

client/src/rpc_sender.rs

@@ -23,13 +23,9 @@ pub struct RpcTransportStats {
 /// `RpcSender` implements the underlying transport of requests to, and
 /// responses from, a Solana node, and is used primarily by [`RpcClient`].
 ///
-/// It is typically implemented by [`HttpSender`] in production, and
-/// [`MockSender`] in unit tests.
-///
-/// [`HttpSender`]: crate::http_sender::HttpSender
-/// [`MockSender`]: crate::mock_sender::MockSender
+/// [`RpcClient`]: crate::rpc_client::RpcClient
 #[async_trait]
-pub(crate) trait RpcSender {
+pub trait RpcSender {
     async fn send(
         &self,
         request: RpcRequest,

client/src/thin_client.rs

@@ -4,8 +4,10 @@
 //! unstable and may change in future releases.
 
 use {
-    crate::{rpc_client::RpcClient, rpc_config::RpcProgramAccountsConfig, rpc_response::Response},
-    bincode::{serialize_into, serialized_size},
+    crate::{
+        rpc_client::RpcClient, rpc_config::RpcProgramAccountsConfig, rpc_response::Response,
+        tpu_connection::TpuConnection, udp_client::UdpTpuConnection,
+    },
     log::*,
     solana_sdk::{
         account::Account,
@@ -17,13 +19,12 @@ use {
         hash::Hash,
         instruction::Instruction,
         message::Message,
-        packet::PACKET_DATA_SIZE,
         pubkey::Pubkey,
         signature::{Keypair, Signature, Signer},
         signers::Signers,
         system_instruction,
         timing::duration_as_ms,
-        transaction::{self, Transaction},
+        transaction::{self, Transaction, VersionedTransaction},
         transport::Result as TransportResult,
     },
     std::{
@@ -117,22 +118,20 @@ impl ClientOptimizer {
 }
 
 /// An object for querying and sending transactions to the network.
-pub struct ThinClient {
-    transactions_socket: UdpSocket,
-    tpu_addrs: Vec<SocketAddr>,
+pub struct ThinClient<C: 'static + TpuConnection> {
     rpc_clients: Vec<RpcClient>,
+    tpu_connections: Vec<C>,
     optimizer: ClientOptimizer,
 }
 
-impl ThinClient {
+impl<C: 'static + TpuConnection> ThinClient<C> {
     /// Create a new ThinClient that will interface with the Rpc at `rpc_addr` using TCP
-    /// and the Tpu at `tpu_addr` over `transactions_socket` using UDP.
+    /// and the Tpu at `tpu_addr` over `transactions_socket` using Quic or UDP
+    /// (currently hardcoded to UDP)
     pub fn new(rpc_addr: SocketAddr, tpu_addr: SocketAddr, transactions_socket: UdpSocket) -> Self {
-        Self::new_from_client(
-            tpu_addr,
-            transactions_socket,
-            RpcClient::new_socket(rpc_addr),
-        )
+        let tpu_connection = C::new(transactions_socket, tpu_addr);
+
+        Self::new_from_client(RpcClient::new_socket(rpc_addr), tpu_connection)
     }
 
     pub fn new_socket_with_timeout(
@@ -142,18 +141,14 @@ impl ThinClient {
         timeout: Duration,
     ) -> Self {
         let rpc_client = RpcClient::new_socket_with_timeout(rpc_addr, timeout);
-        Self::new_from_client(tpu_addr, transactions_socket, rpc_client)
+        let tpu_connection = C::new(transactions_socket, tpu_addr);
+        Self::new_from_client(rpc_client, tpu_connection)
     }
 
-    fn new_from_client(
-        tpu_addr: SocketAddr,
-        transactions_socket: UdpSocket,
-        rpc_client: RpcClient,
-    ) -> Self {
+    fn new_from_client(rpc_client: RpcClient, tpu_connection: C) -> Self {
         Self {
-            transactions_socket,
-            tpu_addrs: vec![tpu_addr],
             rpc_clients: vec![rpc_client],
+            tpu_connections: vec![tpu_connection],
             optimizer: ClientOptimizer::new(0),
         }
     }
@@ -168,16 +163,19 @@ impl ThinClient {
 
         let rpc_clients: Vec<_> = rpc_addrs.into_iter().map(RpcClient::new_socket).collect();
         let optimizer = ClientOptimizer::new(rpc_clients.len());
+        let tpu_connections: Vec<_> = tpu_addrs
+            .into_iter()
+            .map(|tpu_addr| C::new(transactions_socket.try_clone().unwrap(), tpu_addr))
+            .collect();
         Self {
-            transactions_socket,
-            tpu_addrs,
             rpc_clients,
+            tpu_connections,
             optimizer,
         }
     }
 
-    fn tpu_addr(&self) -> &SocketAddr {
-        &self.tpu_addrs[self.optimizer.best()]
+    fn tpu_connection(&self) -> &C {
+        &self.tpu_connections[self.optimizer.best()]
     }
 
     fn rpc_client(&self) -> &RpcClient {
@@ -205,7 +203,6 @@ impl ThinClient {
         self.send_and_confirm_transaction(&[keypair], transaction, tries, 0)
     }
 
-    /// Retry sending a signed Transaction to the server for processing
     pub fn send_and_confirm_transaction<T: Signers>(
         &self,
         keypairs: &T,
@@ -215,18 +212,16 @@ impl ThinClient {
     ) -> TransportResult<Signature> {
         for x in 0..tries {
             let now = Instant::now();
-            let mut buf = vec![0; serialized_size(&transaction).unwrap() as usize];
-            let mut wr = std::io::Cursor::new(&mut buf[..]);
             let mut num_confirmed = 0;
             let mut wait_time = MAX_PROCESSING_AGE;
-            serialize_into(&mut wr, &transaction)
-                .expect("serialize Transaction in pub fn transfer_signed");
             // resend the same transaction until the transaction has no chance of succeeding
+            let wire_transaction =
+                bincode::serialize(&transaction).expect("transaction serialization failed");
             while now.elapsed().as_secs() < wait_time as u64 {
                 if num_confirmed == 0 {
                     // Send the transaction if there has been no confirmation (e.g. the first time)
-                    self.transactions_socket
-                        .send_to(&buf[..], &self.tpu_addr())?;
+                    self.tpu_connection()
+                        .send_wire_transaction(&wire_transaction)?;
                 }
 
                 if let Ok(confirmed_blocks) = self.poll_for_signature_confirmation(
@@ -321,13 +316,13 @@ impl ThinClient {
     }
 }
 
-impl Client for ThinClient {
+impl<C: 'static + TpuConnection> Client for ThinClient<C> {
     fn tpu_addr(&self) -> String {
-        self.tpu_addr().to_string()
+        self.tpu_connection().tpu_addr().to_string()
     }
 }
 
-impl SyncClient for ThinClient {
+impl<C: 'static + TpuConnection> SyncClient for ThinClient<C> {
     fn send_and_confirm_message<T: Signers>(
         &self,
         keypairs: &T,
@@ -607,17 +602,21 @@ impl SyncClient for ThinClient {
     }
 }
 
-impl AsyncClient for ThinClient {
+impl<C: 'static + TpuConnection> AsyncClient for ThinClient<C> {
     fn async_send_transaction(&self, transaction: Transaction) -> TransportResult<Signature> {
-        let mut buf = vec![0; serialized_size(&transaction).unwrap() as usize];
-        let mut wr = std::io::Cursor::new(&mut buf[..]);
-        serialize_into(&mut wr, &transaction)
-            .expect("serialize Transaction in pub fn transfer_signed");
-        assert!(buf.len() < PACKET_DATA_SIZE);
-        self.transactions_socket
-            .send_to(&buf[..], &self.tpu_addr())?;
+        let transaction = VersionedTransaction::from(transaction);
+        self.tpu_connection()
+            .serialize_and_send_transaction(&transaction)?;
         Ok(transaction.signatures[0])
     }
+
+    fn async_send_batch(&self, transactions: Vec<Transaction>) -> TransportResult<()> {
+        let batch: Vec<VersionedTransaction> = transactions.into_iter().map(Into::into).collect();
+        self.tpu_connection()
+            .par_serialize_and_send_transaction_batch(&batch[..])?;
+        Ok(())
+    }
+
     fn async_send_message<T: Signers>(
         &self,
         keypairs: &T,
@@ -649,20 +648,23 @@ impl AsyncClient for ThinClient {
     }
 }
 
-pub fn create_client((rpc, tpu): (SocketAddr, SocketAddr), range: (u16, u16)) -> ThinClient {
+pub fn create_client(
+    (rpc, tpu): (SocketAddr, SocketAddr),
+    range: (u16, u16),
+) -> ThinClient<UdpTpuConnection> {
     let (_, transactions_socket) =
         solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), range).unwrap();
-    ThinClient::new(rpc, tpu, transactions_socket)
+    ThinClient::<UdpTpuConnection>::new(rpc, tpu, transactions_socket)
 }
 
 pub fn create_client_with_timeout(
     (rpc, tpu): (SocketAddr, SocketAddr),
     range: (u16, u16),
     timeout: Duration,
-) -> ThinClient {
+) -> ThinClient<UdpTpuConnection> {
     let (_, transactions_socket) =
         solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), range).unwrap();
-    ThinClient::new_socket_with_timeout(rpc, tpu, transactions_socket, timeout)
+    ThinClient::<UdpTpuConnection>::new_socket_with_timeout(rpc, tpu, transactions_socket, timeout)
 }
 
 #[cfg(test)]

client/src/tpu_connection.rs

@@ -0,0 +1,40 @@
+use {
+    rayon::iter::{IntoParallelIterator, ParallelIterator},
+    solana_sdk::{transaction::VersionedTransaction, transport::Result as TransportResult},
+    std::net::{SocketAddr, UdpSocket},
+};
+
+pub trait TpuConnection {
+    fn new(client_socket: UdpSocket, tpu_addr: SocketAddr) -> Self;
+
+    fn tpu_addr(&self) -> &SocketAddr;
+
+    fn serialize_and_send_transaction(
+        &self,
+        transaction: &VersionedTransaction,
+    ) -> TransportResult<()> {
+        let wire_transaction =
+            bincode::serialize(transaction).expect("serialize Transaction in send_batch");
+        self.send_wire_transaction(&wire_transaction)
+    }
+
+    fn send_wire_transaction<T>(&self, wire_transaction: T) -> TransportResult<()>
+    where
+        T: AsRef<[u8]>;
+
+    fn par_serialize_and_send_transaction_batch(
+        &self,
+        transactions: &[VersionedTransaction],
+    ) -> TransportResult<()> {
+        let buffers = transactions
+            .into_par_iter()
+            .map(|tx| bincode::serialize(&tx).expect("serialize Transaction in send_batch"))
+            .collect::<Vec<_>>();
+
+        self.send_wire_transaction_batch(&buffers)
+    }
+
+    fn send_wire_transaction_batch<T>(&self, buffers: &[T]) -> TransportResult<()>
+    where
+        T: AsRef<[u8]>;
+}

client/src/udp_client.rs

@@ -0,0 +1,45 @@
+//! Simple TPU client that communicates with the given UDP port with UDP and provides
+//! an interface for sending transactions
+
+use {
+    crate::tpu_connection::TpuConnection,
+    core::iter::repeat,
+    solana_sdk::transport::Result as TransportResult,
+    solana_streamer::sendmmsg::batch_send,
+    std::net::{SocketAddr, UdpSocket},
+};
+
+pub struct UdpTpuConnection {
+    socket: UdpSocket,
+    addr: SocketAddr,
+}
+
+impl TpuConnection for UdpTpuConnection {
+    fn new(client_socket: UdpSocket, tpu_addr: SocketAddr) -> Self {
+        Self {
+            socket: client_socket,
+            addr: tpu_addr,
+        }
+    }
+
+    fn tpu_addr(&self) -> &SocketAddr {
+        &self.addr
+    }
+
+    fn send_wire_transaction<T>(&self, wire_transaction: T) -> TransportResult<()>
+    where
+        T: AsRef<[u8]>,
+    {
+        self.socket.send_to(wire_transaction.as_ref(), self.addr)?;
+        Ok(())
+    }
+
+    fn send_wire_transaction_batch<T>(&self, buffers: &[T]) -> TransportResult<()>
+    where
+        T: AsRef<[u8]>,
+    {
+        let pkts: Vec<_> = buffers.iter().zip(repeat(self.tpu_addr())).collect();
+        batch_send(&self.socket, &pkts)?;
+        Ok(())
+    }
+}

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.10.0"
+version = "1.10.6"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-core"
 readme = "../README.md"
@@ -15,73 +15,69 @@ codecov = { repository = "solana-labs/solana", branch = "master", service = "git
 
 [dependencies]
 ahash = "0.7.6"
-base64 = "0.12.3"
+base64 = "0.13.0"
 bincode = "1.3.3"
 bs58 = "0.4.0"
 chrono = { version = "0.4.11", features = ["serde"] }
 crossbeam-channel = "0.5"
 dashmap = { version = "4.0.2", features = ["rayon", "raw-api"] }
-etcd-client = { version = "0.8.3", features = ["tls"]}
+etcd-client = { version = "0.8.4", features = ["tls"] }
 fs_extra = "1.2.0"
 histogram = "0.6.9"
 itertools = "0.10.3"
 log = "0.4.14"
-lru = "0.7.2"
+lru = "0.7.3"
 rand = "0.7.0"
 rand_chacha = "0.2.2"
 rayon = "1.5.1"
-retain_mut = "0.1.5"
+retain_mut = "0.1.7"
 serde = "1.0.136"
 serde_derive = "1.0.103"
-solana-address-lookup-table-program = { path = "../programs/address-lookup-table", version = "=1.10.0" }
-solana-bloom = { path = "../bloom", version = "=1.10.0" }
-solana-accountsdb-plugin-manager = { path = "../accountsdb-plugin-manager", version = "=1.10.0" }
-solana-client = { path = "../client", version = "=1.10.0" }
-solana-entry = { path = "../entry", version = "=1.10.0" }
-solana-gossip = { path = "../gossip", version = "=1.10.0" }
-solana-ledger = { path = "../ledger", version = "=1.10.0" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-solana-metrics = { path = "../metrics", version = "=1.10.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
-solana-perf = { path = "../perf", version = "=1.10.0" }
-solana-poh = { path = "../poh", version = "=1.10.0" }
-solana-program-runtime = { path = "../program-runtime", version = "=1.10.0" }
-solana-rpc = { path = "../rpc", version = "=1.10.0" }
-solana-replica-lib = { path = "../replica-lib", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-frozen-abi = { path = "../frozen-abi", version = "=1.10.0" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.10.0" }
-solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.10.0" }
-solana-streamer = { path = "../streamer", version = "=1.10.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.10.0" }
+solana-address-lookup-table-program = { path = "../programs/address-lookup-table", version = "=1.10.6" }
+solana-bloom = { path = "../bloom", version = "=1.10.6" }
+solana-client = { path = "../client", version = "=1.10.6" }
+solana-entry = { path = "../entry", version = "=1.10.6" }
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.10.6" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.10.6" }
+solana-geyser-plugin-manager = { path = "../geyser-plugin-manager", version = "=1.10.6" }
+solana-gossip = { path = "../gossip", version = "=1.10.6" }
+solana-ledger = { path = "../ledger", version = "=1.10.6" }
+solana-measure = { path = "../measure", version = "=1.10.6" }
+solana-metrics = { path = "../metrics", version = "=1.10.6" }
+solana-net-utils = { path = "../net-utils", version = "=1.10.6" }
+solana-perf = { path = "../perf", version = "=1.10.6" }
+solana-poh = { path = "../poh", version = "=1.10.6" }
+solana-program-runtime = { path = "../program-runtime", version = "=1.10.6" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.10.6" }
+solana-replica-lib = { path = "../replica-lib", version = "=1.10.6" }
+solana-rpc = { path = "../rpc", version = "=1.10.6" }
+solana-runtime = { path = "../runtime", version = "=1.10.6" }
+solana-sdk = { path = "../sdk", version = "=1.10.6" }
+solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.10.6" }
+solana-streamer = { path = "../streamer", version = "=1.10.6" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.6" }
+solana-vote-program = { path = "../programs/vote", version = "=1.10.6" }
+sys-info = "0.9.1"
 tempfile = "3.3.0"
 thiserror = "1.0"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.10.0" }
-sys-info = "0.9.1"
 tokio = { version = "1", features = ["full"] }
 trees = "0.4.2"
 
 [dev-dependencies]
-jsonrpc-core = "18.0.0"
-jsonrpc-core-client = { version = "18.0.0", features = ["ipc", "ws"] }
-jsonrpc-derive = "18.0.0"
-jsonrpc-pubsub = "18.0.0"
 matches = "0.1.9"
 raptorq = "1.6.5"
-reqwest = { version = "0.11.9", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-serde_json = "1.0.78"
-serial_test = "0.5.1"
-solana-logger = { path = "../logger", version = "=1.10.0" }
-solana-program-runtime = { path = "../program-runtime", version = "=1.10.0" }
-solana-stake-program = { path = "../programs/stake", version = "=1.10.0" }
-solana-version = { path = "../version", version = "=1.10.0" }
+reqwest = { version = "0.11.10", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+serde_json = "1.0.79"
+serial_test = "0.6.0"
+solana-logger = { path = "../logger", version = "=1.10.6" }
+solana-program-runtime = { path = "../program-runtime", version = "=1.10.6" }
+solana-stake-program = { path = "../programs/stake", version = "=1.10.6" }
+solana-version = { path = "../version", version = "=1.10.6" }
 static_assertions = "1.1.0"
 systemstat = "0.1.10"
 
 [target."cfg(unix)".dependencies]
-sysctl = "0.4.3"
+sysctl = "0.4.4"
 
 [build-dependencies]
 rustc_version = "0.4"

core/benches/banking_stage.rs

@@ -12,6 +12,7 @@ use {
         banking_stage::{BankingStage, BankingStageStats},
         leader_slot_banking_stage_metrics::LeaderSlotMetricsTracker,
         qos_service::QosService,
+        unprocessed_packet_batches::*,
     },
     solana_entry::entry::{next_hash, Entry},
     solana_gossip::cluster_info::{ClusterInfo, Node},
@@ -36,7 +37,6 @@ use {
     },
     solana_streamer::socket::SocketAddrSpace,
     std::{
-        collections::VecDeque,
         sync::{atomic::Ordering, Arc, RwLock},
         time::{Duration, Instant},
     },
@@ -79,10 +79,14 @@ fn bench_consume_buffered(bencher: &mut Bencher) {
         let len = 4096;
         let chunk_size = 1024;
         let batches = to_packet_batches(&vec![tx; len], chunk_size);
-        let mut packet_batches = VecDeque::new();
+        let mut packet_batches = UnprocessedPacketBatches::new();
         for batch in batches {
             let batch_len = batch.packets.len();
-            packet_batches.push_back((batch, vec![0usize; batch_len], false));
+            packet_batches.push_back(DeserializedPacketBatch::new(
+                batch,
+                vec![0usize; batch_len],
+                false,
+            ));
         }
         let (s, _r) = unbounded();
         // This tests the performance of buffering packets.

core/benches/cluster_nodes.rs

@@ -0,0 +1,112 @@
+#![feature(test)]
+
+extern crate test;
+
+use {
+    rand::{seq::SliceRandom, Rng},
+    solana_core::{
+        cluster_nodes::{make_test_cluster, new_cluster_nodes, ClusterNodes},
+        retransmit_stage::RetransmitStage,
+    },
+    solana_gossip::contact_info::ContactInfo,
+    solana_sdk::{clock::Slot, hash::hashv, pubkey::Pubkey, signature::Signature},
+    test::Bencher,
+};
+
+const NUM_SIMULATED_SHREDS: usize = 4;
+
+fn make_cluster_nodes<R: Rng>(
+    rng: &mut R,
+    unstaked_ratio: Option<(u32, u32)>,
+) -> (Vec<ContactInfo>, ClusterNodes<RetransmitStage>) {
+    let (nodes, stakes, cluster_info) = make_test_cluster(rng, 5_000, unstaked_ratio);
+    let cluster_nodes = new_cluster_nodes::<RetransmitStage>(&cluster_info, &stakes);
+    (nodes, cluster_nodes)
+}
+
+fn get_retransmit_peers_deterministic(
+    cluster_nodes: &ClusterNodes<RetransmitStage>,
+    slot: &Slot,
+    slot_leader: &Pubkey,
+    num_simulated_shreds: usize,
+) {
+    for i in 0..num_simulated_shreds {
+        // see Shred::seed
+        let shred_seed = hashv(&[
+            &slot.to_le_bytes(),
+            &(i as u32).to_le_bytes(),
+            &slot_leader.to_bytes(),
+        ])
+        .to_bytes();
+
+        let (_neighbors, _children) = cluster_nodes.get_retransmit_peers_deterministic(
+            shred_seed,
+            solana_gossip::cluster_info::DATA_PLANE_FANOUT,
+            *slot_leader,
+        );
+    }
+}
+
+fn get_retransmit_peers_compat(
+    cluster_nodes: &ClusterNodes<RetransmitStage>,
+    slot_leader: &Pubkey,
+    signatures: &[Signature],
+) {
+    for signature in signatures.iter() {
+        // see Shred::seed
+        let signature = signature.as_ref();
+        let offset = signature.len().checked_sub(32).unwrap();
+        let shred_seed = signature[offset..].try_into().unwrap();
+
+        let (_neighbors, _children) = cluster_nodes.get_retransmit_peers_compat(
+            shred_seed,
+            solana_gossip::cluster_info::DATA_PLANE_FANOUT,
+            *slot_leader,
+        );
+    }
+}
+
+fn get_retransmit_peers_deterministic_wrapper(b: &mut Bencher, unstaked_ratio: Option<(u32, u32)>) {
+    let mut rng = rand::thread_rng();
+    let (nodes, cluster_nodes) = make_cluster_nodes(&mut rng, unstaked_ratio);
+    let slot_leader = nodes[1..].choose(&mut rng).unwrap().id;
+    let slot = rand::random::<u64>();
+    b.iter(|| {
+        get_retransmit_peers_deterministic(
+            &cluster_nodes,
+            &slot,
+            &slot_leader,
+            NUM_SIMULATED_SHREDS,
+        )
+    });
+}
+
+fn get_retransmit_peers_compat_wrapper(b: &mut Bencher, unstaked_ratio: Option<(u32, u32)>) {
+    let mut rng = rand::thread_rng();
+    let (nodes, cluster_nodes) = make_cluster_nodes(&mut rng, unstaked_ratio);
+    let slot_leader = nodes[1..].choose(&mut rng).unwrap().id;
+    let signatures: Vec<_> = std::iter::repeat_with(Signature::new_unique)
+        .take(NUM_SIMULATED_SHREDS)
+        .collect();
+    b.iter(|| get_retransmit_peers_compat(&cluster_nodes, &slot_leader, &signatures));
+}
+
+#[bench]
+fn bench_get_retransmit_peers_deterministic_unstaked_ratio_1_2(b: &mut Bencher) {
+    get_retransmit_peers_deterministic_wrapper(b, Some((1, 2)));
+}
+
+#[bench]
+fn bench_get_retransmit_peers_compat_unstaked_ratio_1_2(b: &mut Bencher) {
+    get_retransmit_peers_compat_wrapper(b, Some((1, 2)));
+}
+
+#[bench]
+fn bench_get_retransmit_peers_deterministic_unstaked_ratio_1_32(b: &mut Bencher) {
+    get_retransmit_peers_deterministic_wrapper(b, Some((1, 32)));
+}
+
+#[bench]
+fn bench_get_retransmit_peers_compat_unstaked_ratio_1_32(b: &mut Bencher) {
+    get_retransmit_peers_compat_wrapper(b, Some((1, 32)));
+}

core/benches/sigverify_stage.rs

@@ -159,7 +159,7 @@ fn bench_sigverify_stage(bencher: &mut Bencher) {
         for _ in 0..batches.len() {
             if let Some(batch) = batches.pop() {
                 sent_len += batch.packets.len();
-                packet_s.send(batch).unwrap();
+                packet_s.send(vec![batch]).unwrap();
             }
         }
         let mut received = 0;

core/src/ancestor_hashes_service.rs

@@ -3,6 +3,7 @@ use {
         cluster_slots::ClusterSlots,
         duplicate_repair_status::{DeadSlotAncestorRequestStatus, DuplicateAncestorDecision},
         outstanding_requests::OutstandingRequests,
+        packet_threshold::DynamicPacketToProcessThreshold,
         repair_response::{self},
         repair_service::{DuplicateSlotsResetSender, RepairInfo, RepairStatsGroup},
         replay_stage::DUPLICATE_THRESHOLD,
@@ -12,7 +13,6 @@ use {
     crossbeam_channel::{unbounded, Receiver, Sender},
     dashmap::{mapref::entry::Entry::Occupied, DashMap},
     solana_ledger::{blockstore::Blockstore, shred::SIZE_OF_NONCE},
-    solana_measure::measure::Measure,
     solana_perf::{
         packet::{limited_deserialize, Packet, PacketBatch},
         recycler::Recycler,
@@ -208,7 +208,7 @@ impl AncestorHashesService {
             .spawn(move || {
                 let mut last_stats_report = Instant::now();
                 let mut stats = AncestorHashesResponsesStats::default();
-                let mut max_packets = 1024;
+                let mut packet_threshold = DynamicPacketToProcessThreshold::default();
                 loop {
                     let result = Self::process_new_packets_from_channel(
                         &ancestor_hashes_request_statuses,
@@ -216,13 +216,13 @@ impl AncestorHashesService {
                         &blockstore,
                         &outstanding_requests,
                         &mut stats,
-                        &mut max_packets,
+                        &mut packet_threshold,
                         &duplicate_slots_reset_sender,
                         &retryable_slots_sender,
                     );
                     match result {
                         Err(Error::RecvTimeout(_)) | Ok(_) => {}
-                        Err(err) => info!("ancestors hashes reponses listener error: {:?}", err),
+                        Err(err) => info!("ancestors hashes responses listener error: {:?}", err),
                     };
                     if exit.load(Ordering::Relaxed) {
                         return;
@@ -243,7 +243,7 @@ impl AncestorHashesService {
         blockstore: &Blockstore,
         outstanding_requests: &RwLock<OutstandingAncestorHashesRepairs>,
         stats: &mut AncestorHashesResponsesStats,
-        max_packets: &mut usize,
+        packet_threshold: &mut DynamicPacketToProcessThreshold,
         duplicate_slots_reset_sender: &DuplicateSlotsResetSender,
         retryable_slots_sender: &RetryableSlotsSender,
     ) -> Result<()> {
@@ -254,18 +254,17 @@ impl AncestorHashesService {
         let mut dropped_packets = 0;
         while let Ok(batch) = response_receiver.try_recv() {
             total_packets += batch.packets.len();
-            if total_packets < *max_packets {
-                // Drop the rest in the channel in case of DOS
-                packet_batches.push(batch);
-            } else {
+            if packet_threshold.should_drop(total_packets) {
                 dropped_packets += batch.packets.len();
+            } else {
+                packet_batches.push(batch);
             }
         }
 
         stats.dropped_packets += dropped_packets;
         stats.total_packets += total_packets;
 
-        let mut time = Measure::start("ancestor_hashes::handle_packets");
+        let timer = Instant::now();
         for packet_batch in packet_batches {
             Self::process_packet_batch(
                 ancestor_hashes_request_statuses,
@@ -277,14 +276,7 @@ impl AncestorHashesService {
                 retryable_slots_sender,
             );
         }
-        time.stop();
-        if total_packets >= *max_packets {
-            if time.as_ms() > 1000 {
-                *max_packets = (*max_packets * 9) / 10;
-            } else {
-                *max_packets = (*max_packets * 10) / 9;
-            }
-        }
+        packet_threshold.update(total_packets, timer.elapsed());
         Ok(())
     }
 

core/src/cluster_nodes.rs

@@ -2,12 +2,14 @@ use {
     crate::{broadcast_stage::BroadcastStage, retransmit_stage::RetransmitStage},
     itertools::Itertools,
     lru::LruCache,
-    rand::SeedableRng,
+    rand::{seq::SliceRandom, Rng, SeedableRng},
     rand_chacha::ChaChaRng,
     solana_gossip::{
         cluster_info::{compute_retransmit_peers, ClusterInfo},
         contact_info::ContactInfo,
+        crds::GossipRoute,
         crds_gossip_pull::CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS,
+        crds_value::{CrdsData, CrdsValue},
         weighted_shuffle::{weighted_best, weighted_shuffle, WeightedShuffle},
     },
     solana_ledger::shred::Shred,
@@ -16,6 +18,7 @@ use {
         clock::{Epoch, Slot},
         feature_set,
         pubkey::Pubkey,
+        signature::Keypair,
         timing::timestamp,
     },
     solana_streamer::socket::SocketAddrSpace,
@@ -23,6 +26,7 @@ use {
         any::TypeId,
         cmp::Reverse,
         collections::HashMap,
+        iter::repeat_with,
         marker::PhantomData,
         net::SocketAddr,
         ops::Deref,
@@ -39,7 +43,7 @@ enum NodeId {
     Pubkey(Pubkey),
 }
 
-struct Node {
+pub struct Node {
     node: NodeId,
     stake: u64,
 }
@@ -233,6 +237,18 @@ impl ClusterNodes<RetransmitStage> {
         if !enable_turbine_peers_shuffle_patch(shred.slot(), root_bank) {
             return self.get_retransmit_peers_compat(shred_seed, fanout, slot_leader);
         }
+        self.get_retransmit_peers_deterministic(shred_seed, fanout, slot_leader)
+    }
+
+    pub fn get_retransmit_peers_deterministic(
+        &self,
+        shred_seed: [u8; 32],
+        fanout: usize,
+        slot_leader: Pubkey,
+    ) -> (
+        Vec<&Node>, // neighbors
+        Vec<&Node>, // children
+    ) {
         let mut weighted_shuffle = self.weighted_shuffle.clone();
         // Exclude slot leader from list of nodes.
         if slot_leader == self.pubkey {
@@ -256,7 +272,7 @@ impl ClusterNodes<RetransmitStage> {
         (neighbors, children)
     }
 
-    fn get_retransmit_peers_compat(
+    pub fn get_retransmit_peers_compat(
         &self,
         shred_seed: [u8; 32],
         fanout: usize,
@@ -297,7 +313,7 @@ impl ClusterNodes<RetransmitStage> {
     }
 }
 
-fn new_cluster_nodes<T: 'static>(
+pub fn new_cluster_nodes<T: 'static>(
     cluster_info: &ClusterInfo,
     stakes: &HashMap<Pubkey, u64>,
 ) -> ClusterNodes<T> {
@@ -310,7 +326,7 @@ fn new_cluster_nodes<T: 'static>(
         .collect();
     let broadcast = TypeId::of::<T>() == TypeId::of::<BroadcastStage>();
     let stakes: Vec<u64> = nodes.iter().map(|node| node.stake).collect();
-    let mut weighted_shuffle = WeightedShuffle::new(&stakes).unwrap();
+    let mut weighted_shuffle = WeightedShuffle::new("cluster-nodes", &stakes);
     if broadcast {
         weighted_shuffle.remove_index(index[&self_pubkey]);
     }
@@ -462,22 +478,61 @@ impl From<Pubkey> for NodeId {
     }
 }
 
+pub fn make_test_cluster<R: Rng>(
+    rng: &mut R,
+    num_nodes: usize,
+    unstaked_ratio: Option<(u32, u32)>,
+) -> (
+    Vec<ContactInfo>,
+    HashMap<Pubkey, u64>, // stakes
+    ClusterInfo,
+) {
+    let (unstaked_numerator, unstaked_denominator) = unstaked_ratio.unwrap_or((1, 7));
+    let mut nodes: Vec<_> = repeat_with(|| ContactInfo::new_rand(rng, None))
+        .take(num_nodes)
+        .collect();
+    nodes.shuffle(rng);
+    let this_node = nodes[0].clone();
+    let mut stakes: HashMap<Pubkey, u64> = nodes
+        .iter()
+        .filter_map(|node| {
+            if rng.gen_ratio(unstaked_numerator, unstaked_denominator) {
+                None // No stake for some of the nodes.
+            } else {
+                Some((node.id, rng.gen_range(0, 20)))
+            }
+        })
+        .collect();
+    // Add some staked nodes with no contact-info.
+    stakes.extend(repeat_with(|| (Pubkey::new_unique(), rng.gen_range(0, 20))).take(100));
+    let cluster_info = ClusterInfo::new(
+        this_node,
+        Arc::new(Keypair::new()),
+        SocketAddrSpace::Unspecified,
+    );
+    {
+        let now = timestamp();
+        let mut gossip_crds = cluster_info.gossip.crds.write().unwrap();
+        // First node is pushed to crds table by ClusterInfo constructor.
+        for node in nodes.iter().skip(1) {
+            let node = CrdsData::ContactInfo(node.clone());
+            let node = CrdsValue::new_unsigned(node);
+            assert_eq!(
+                gossip_crds.insert(node, now, GossipRoute::LocalMessage),
+                Ok(())
+            );
+        }
+    }
+    (nodes, stakes, cluster_info)
+}
+
 #[cfg(test)]
 mod tests {
     use {
         super::*,
-        rand::{seq::SliceRandom, Rng},
-        solana_gossip::{
-            crds::GossipRoute,
-            crds_value::{CrdsData, CrdsValue},
-            deprecated::{
-                shuffle_peers_and_index, sorted_retransmit_peers_and_stakes,
-                sorted_stakes_with_index,
-            },
+        solana_gossip::deprecated::{
+            shuffle_peers_and_index, sorted_retransmit_peers_and_stakes, sorted_stakes_with_index,
         },
-        solana_sdk::{signature::Keypair, timing::timestamp},
-        solana_streamer::socket::SocketAddrSpace,
-        std::{iter::repeat_with, sync::Arc},
     };
 
     // Legacy methods copied for testing backward compatibility.
@@ -499,55 +554,10 @@ mod tests {
         sorted_stakes_with_index(peers, stakes)
     }
 
-    fn make_cluster<R: Rng>(
-        rng: &mut R,
-    ) -> (
-        Vec<ContactInfo>,
-        HashMap<Pubkey, u64>, // stakes
-        ClusterInfo,
-    ) {
-        let mut nodes: Vec<_> = repeat_with(|| ContactInfo::new_rand(rng, None))
-            .take(1000)
-            .collect();
-        nodes.shuffle(rng);
-        let this_node = nodes[0].clone();
-        let mut stakes: HashMap<Pubkey, u64> = nodes
-            .iter()
-            .filter_map(|node| {
-                if rng.gen_ratio(1, 7) {
-                    None // No stake for some of the nodes.
-                } else {
-                    Some((node.id, rng.gen_range(0, 20)))
-                }
-            })
-            .collect();
-        // Add some staked nodes with no contact-info.
-        stakes.extend(repeat_with(|| (Pubkey::new_unique(), rng.gen_range(0, 20))).take(100));
-        let cluster_info = ClusterInfo::new(
-            this_node,
-            Arc::new(Keypair::new()),
-            SocketAddrSpace::Unspecified,
-        );
-        {
-            let now = timestamp();
-            let mut gossip_crds = cluster_info.gossip.crds.write().unwrap();
-            // First node is pushed to crds table by ClusterInfo constructor.
-            for node in nodes.iter().skip(1) {
-                let node = CrdsData::ContactInfo(node.clone());
-                let node = CrdsValue::new_unsigned(node);
-                assert_eq!(
-                    gossip_crds.insert(node, now, GossipRoute::LocalMessage),
-                    Ok(())
-                );
-            }
-        }
-        (nodes, stakes, cluster_info)
-    }
-
     #[test]
     fn test_cluster_nodes_retransmit() {
         let mut rng = rand::thread_rng();
-        let (nodes, stakes, cluster_info) = make_cluster(&mut rng);
+        let (nodes, stakes, cluster_info) = make_test_cluster(&mut rng, 1_000, None);
         let this_node = cluster_info.my_contact_info();
         // ClusterInfo::tvu_peers excludes the node itself.
         assert_eq!(cluster_info.tvu_peers().len(), nodes.len() - 1);
@@ -628,7 +638,7 @@ mod tests {
     #[test]
     fn test_cluster_nodes_broadcast() {
         let mut rng = rand::thread_rng();
-        let (nodes, stakes, cluster_info) = make_cluster(&mut rng);
+        let (nodes, stakes, cluster_info) = make_test_cluster(&mut rng, 1_000, None);
         // ClusterInfo::tvu_peers excludes the node itself.
         assert_eq!(cluster_info.tvu_peers().len(), nodes.len() - 1);
         let cluster_nodes = ClusterNodes::<BroadcastStage>::new(&cluster_info, &stakes);

core/src/cost_update_service.rs

@@ -16,39 +16,24 @@ use {
     },
 };
 
-// Update blockstore persistence storage when accumulated cost_table updates count exceeds the threshold
-const PERSIST_THRESHOLD: u64 = 1_000;
-
 #[derive(Default)]
 pub struct CostUpdateServiceTiming {
     last_print: u64,
     update_cost_model_count: u64,
     update_cost_model_elapsed: u64,
-    persist_cost_table_elapsed: u64,
 }
 
 impl CostUpdateServiceTiming {
-    fn update(
-        &mut self,
-        update_cost_model_count: Option<u64>,
-        update_cost_model_elapsed: Option<u64>,
-        persist_cost_table_elapsed: Option<u64>,
-    ) {
-        if let Some(update_cost_model_count) = update_cost_model_count {
-            self.update_cost_model_count += update_cost_model_count;
-        }
-        if let Some(update_cost_model_elapsed) = update_cost_model_elapsed {
-            self.update_cost_model_elapsed += update_cost_model_elapsed;
-        }
-        if let Some(persist_cost_table_elapsed) = persist_cost_table_elapsed {
-            self.persist_cost_table_elapsed += persist_cost_table_elapsed;
-        }
+    fn update(&mut self, update_cost_model_count: u64, update_cost_model_elapsed: u64) {
+        self.update_cost_model_count += update_cost_model_count;
+        self.update_cost_model_elapsed += update_cost_model_elapsed;
 
         let now = timestamp();
         let elapsed_ms = now - self.last_print;
         if elapsed_ms > 1000 {
             datapoint_info!(
                 "cost-update-service-stats",
+                ("total_elapsed_us", elapsed_ms * 1000, i64),
                 (
                     "update_cost_model_count",
                     self.update_cost_model_count as i64,
@@ -59,11 +44,6 @@ impl CostUpdateServiceTiming {
                     self.update_cost_model_elapsed as i64,
                     i64
                 ),
-                (
-                    "persist_cost_table_elapsed",
-                    self.persist_cost_table_elapsed as i64,
-                    i64
-                ),
             );
 
             *self = CostUpdateServiceTiming::default();
@@ -109,13 +89,11 @@ impl CostUpdateService {
     }
 
     fn service_loop(
-        blockstore: Arc<Blockstore>,
+        _blockstore: Arc<Blockstore>,
         cost_model: Arc<RwLock<CostModel>>,
         cost_update_receiver: CostUpdateReceiver,
     ) {
         let mut cost_update_service_timing = CostUpdateServiceTiming::default();
-        let mut update_count = 0_u64;
-
         for cost_update in cost_update_receiver.iter() {
             match cost_update {
                 CostUpdate::FrozenBank { bank } => {
@@ -124,33 +102,17 @@ impl CostUpdateService {
                 CostUpdate::ExecuteTiming {
                     mut execute_timings,
                 } => {
-                    let mut update_cost_model_time = Measure::start("update_cost_model_time");
-                    update_count += Self::update_cost_model(&cost_model, &mut execute_timings);
-                    update_cost_model_time.stop();
-                    cost_update_service_timing.update(
-                        Some(update_count),
-                        Some(update_cost_model_time.as_us()),
-                        None,
+                    let (update_count, update_cost_model_time) = Measure::this(
+                        |_| Self::update_cost_model(&cost_model, &mut execute_timings),
+                        (),
+                        "update_cost_model_time",
                     );
-
-                    if update_count > PERSIST_THRESHOLD {
-                        let mut persist_cost_table_time = Measure::start("persist_cost_table_time");
-                        Self::persist_cost_table(&blockstore, &cost_model);
-                        update_count = 0_u64;
-                        persist_cost_table_time.stop();
-                        cost_update_service_timing.update(
-                            None,
-                            None,
-                            Some(persist_cost_table_time.as_us()),
-                        );
-                    }
+                    cost_update_service_timing.update(update_count, update_cost_model_time.as_us());
                 }
             }
         }
     }
 
-    // Normalize `program_timings` with current estimated cost, update instruction_cost table
-    // Returns number of updates applied
     fn update_cost_model(
         cost_model: &RwLock<CostModel>,
         execute_timings: &mut ExecuteTimings,
@@ -171,38 +133,9 @@ impl CostUpdateService {
                 .unwrap()
                 .upsert_instruction_cost(program_id, units);
             update_count += 1;
-            debug!(
-                "After replayed into bank, updated cost for instruction {:?}, update_value {}, pre_aggregated_value {}",
-                program_id, units, current_estimated_program_cost
-            );
         }
         update_count
     }
-
-    // 1. Remove obsolete program entries from persisted table to limit its size
-    // 2. Update persisted program cost. This involves EMA cost calculation at
-    //    execute_cost_table.get_cost()
-    fn persist_cost_table(blockstore: &Blockstore, cost_model: &RwLock<CostModel>) {
-        let db_records = blockstore.read_program_costs().expect("read programs");
-        let cost_model = cost_model.read().unwrap();
-        let active_program_keys = cost_model.get_program_keys();
-
-        // delete records from blockstore if they are no longer in cost_table
-        db_records.iter().for_each(|(pubkey, _)| {
-            if !active_program_keys.contains(&pubkey) {
-                blockstore
-                    .delete_program_cost(pubkey)
-                    .expect("delete old program");
-            }
-        });
-
-        active_program_keys.iter().for_each(|program_id| {
-            let cost = cost_model.find_instruction_cost(program_id);
-            blockstore
-                .write_program_cost(program_id, &cost)
-                .expect("persist program costs to blockstore");
-        });
-    }
 }
 
 #[cfg(test)]
@@ -213,9 +146,10 @@ mod tests {
     fn test_update_cost_model_with_empty_execute_timings() {
         let cost_model = Arc::new(RwLock::new(CostModel::default()));
         let mut empty_execute_timings = ExecuteTimings::default();
+
         assert_eq!(
+            0,
             CostUpdateService::update_cost_model(&cost_model, &mut empty_execute_timings),
-            0
         );
     }
 
@@ -233,7 +167,7 @@ mod tests {
             let accumulated_units: u64 = 100;
             let total_errored_units = 0;
             let count: u32 = 10;
-            expected_cost = accumulated_units / count as u64; // = 10
+            expected_cost = accumulated_units / count as u64;
 
             execute_timings.details.per_program_timings.insert(
                 program_key_1,
@@ -245,9 +179,10 @@ mod tests {
                     total_errored_units,
                 },
             );
-            let update_count =
-                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings);
-            assert_eq!(1, update_count);
+            assert_eq!(
+                1,
+                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings),
+            );
             assert_eq!(
                 expected_cost,
                 cost_model
@@ -262,8 +197,8 @@ mod tests {
             let accumulated_us: u64 = 2000;
             let accumulated_units: u64 = 200;
             let count: u32 = 10;
-            // to expect new cost = (mean + 2 * std) of [10, 20]
-            expected_cost = 13;
+            // to expect new cost is Average(new_value, existing_value)
+            expected_cost = ((accumulated_units / count as u64) + expected_cost) / 2;
 
             execute_timings.details.per_program_timings.insert(
                 program_key_1,
@@ -275,9 +210,10 @@ mod tests {
                     total_errored_units: 0,
                 },
             );
-            let update_count =
-                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings);
-            assert_eq!(1, update_count);
+            assert_eq!(
+                1,
+                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings),
+            );
             assert_eq!(
                 expected_cost,
                 cost_model
@@ -328,9 +264,10 @@ mod tests {
                     total_errored_units: 0,
                 },
             );
-            let update_count =
-                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings);
-            assert_eq!(1, update_count);
+            assert_eq!(
+                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings),
+                1
+            );
             assert_eq!(
                 current_program_cost,
                 cost_model
@@ -344,12 +281,8 @@ mod tests {
         // greater than the current instruction cost for the program. Should update with the
         // new erroring compute costs
         let cost_per_error = 1000;
-        // expected_cost = (mean + 2*std) of data points:
-        // [
-        //  100,  // original program_cost
-        //  1000, // cost_per_error
-        // ]
-        let expected_cost = 289u64;
+        // the expect cost is (previous_cost + new_cost)/2 = (100 + 1000)/2 = 550
+        let expected_units = 550;
         {
             let errored_txs_compute_consumed = vec![cost_per_error; 3];
             let total_errored_units = errored_txs_compute_consumed.iter().sum();
@@ -363,12 +296,12 @@ mod tests {
                     total_errored_units,
                 },
             );
-            let update_count =
-                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings);
-
-            assert_eq!(1, update_count);
             assert_eq!(
-                expected_cost,
+                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings),
+                1
+            );
+            assert_eq!(
+                expected_units,
                 cost_model
                     .read()
                     .unwrap()
@@ -379,7 +312,7 @@ mod tests {
         // Test updating cost model with only erroring compute costs where the error cost is
         // `smaller_cost_per_error`, less than the current instruction cost for the program.
         // The cost should not decrease for these new lesser errors
-        let smaller_cost_per_error = expected_cost - 10;
+        let smaller_cost_per_error = expected_units - 10;
         {
             let errored_txs_compute_consumed = vec![smaller_cost_per_error; 3];
             let total_errored_units = errored_txs_compute_consumed.iter().sum();
@@ -393,19 +326,12 @@ mod tests {
                     total_errored_units,
                 },
             );
-            let update_count =
-                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings);
-
-            // expected_cost = (mean = 2*std) of data points:
-            // [
-            //  100,  // original program cost,
-            //  1000, // cost_per_error from above test
-            //  289, // the smaller_cost_per_error will be coalesced to prev cost
-            // ]
-            let expected_cost = 293u64;
-            assert_eq!(1, update_count);
             assert_eq!(
-                expected_cost,
+                CostUpdateService::update_cost_model(&cost_model, &mut execute_timings),
+                1
+            );
+            assert_eq!(
+                expected_units,
                 cost_model
                     .read()
                     .unwrap()

core/src/find_packet_sender_stake_stage.rs

@@ -0,0 +1,185 @@
+use {
+    crossbeam_channel::{Receiver, RecvTimeoutError, Sender},
+    rayon::{prelude::*, ThreadPool},
+    solana_gossip::cluster_info::ClusterInfo,
+    solana_measure::measure::Measure,
+    solana_perf::packet::PacketBatch,
+    solana_rayon_threadlimit::get_thread_count,
+    solana_runtime::bank_forks::BankForks,
+    solana_sdk::timing::timestamp,
+    solana_streamer::streamer::{self, StreamerError},
+    std::{
+        cell::RefCell,
+        collections::HashMap,
+        net::IpAddr,
+        sync::{Arc, RwLock},
+        thread::{self, Builder, JoinHandle},
+        time::{Duration, Instant},
+    },
+};
+
+const IP_TO_STAKE_REFRESH_DURATION: Duration = Duration::from_secs(5);
+
+thread_local!(static PAR_THREAD_POOL: RefCell<ThreadPool> = RefCell::new(rayon::ThreadPoolBuilder::new()
+                    .num_threads(get_thread_count())
+                    .thread_name(|ix| format!("transaction_sender_stake_stage_{}", ix))
+                    .build()
+                    .unwrap()));
+
+pub type FindPacketSenderStakeSender = Sender<Vec<PacketBatch>>;
+pub type FindPacketSenderStakeReceiver = Receiver<Vec<PacketBatch>>;
+
+#[derive(Debug, Default)]
+struct FindPacketSenderStakeStats {
+    last_print: u64,
+    refresh_ip_to_stake_time: u64,
+    apply_sender_stakes_time: u64,
+    send_batches_time: u64,
+    receive_batches_time: u64,
+    total_batches: u64,
+    total_packets: u64,
+}
+
+impl FindPacketSenderStakeStats {
+    fn report(&mut self) {
+        let now = timestamp();
+        let elapsed_ms = now - self.last_print;
+        if elapsed_ms > 2000 {
+            datapoint_info!(
+                "find_packet_sender_stake-services_stats",
+                (
+                    "refresh_ip_to_stake_time",
+                    self.refresh_ip_to_stake_time as i64,
+                    i64
+                ),
+                (
+                    "apply_sender_stakes_time",
+                    self.apply_sender_stakes_time as i64,
+                    i64
+                ),
+                ("send_batches_time", self.send_batches_time as i64, i64),
+                (
+                    "receive_batches_time",
+                    self.receive_batches_time as i64,
+                    i64
+                ),
+                ("total_batches", self.total_batches as i64, i64),
+                ("total_packets", self.total_packets as i64, i64),
+            );
+            *self = FindPacketSenderStakeStats::default();
+            self.last_print = now;
+        }
+    }
+}
+
+pub struct FindPacketSenderStakeStage {
+    thread_hdl: JoinHandle<()>,
+}
+
+impl FindPacketSenderStakeStage {
+    pub fn new(
+        packet_receiver: streamer::PacketBatchReceiver,
+        sender: FindPacketSenderStakeSender,
+        bank_forks: Arc<RwLock<BankForks>>,
+        cluster_info: Arc<ClusterInfo>,
+    ) -> Self {
+        let mut stats = FindPacketSenderStakeStats::default();
+        let thread_hdl = Builder::new()
+            .name("find-packet-sender-stake".to_string())
+            .spawn(move || {
+                let mut last_stakes = Instant::now();
+                let mut ip_to_stake: HashMap<IpAddr, u64> = HashMap::new();
+                loop {
+                    let mut refresh_ip_to_stake_time = Measure::start("refresh_ip_to_stake_time");
+                    Self::try_refresh_ip_to_stake(
+                        &mut last_stakes,
+                        &mut ip_to_stake,
+                        bank_forks.clone(),
+                        cluster_info.clone(),
+                    );
+                    refresh_ip_to_stake_time.stop();
+                    stats.refresh_ip_to_stake_time = stats
+                        .refresh_ip_to_stake_time
+                        .saturating_add(refresh_ip_to_stake_time.as_us());
+
+                    match streamer::recv_packet_batches(&packet_receiver) {
+                        Ok((mut batches, num_packets, recv_duration)) => {
+                            let num_batches = batches.len();
+                            let mut apply_sender_stakes_time =
+                                Measure::start("apply_sender_stakes_time");
+                            Self::apply_sender_stakes(&mut batches, &ip_to_stake);
+                            apply_sender_stakes_time.stop();
+
+                            let mut send_batches_time = Measure::start("send_batches_time");
+                            if let Err(e) = sender.send(batches) {
+                                info!("Sender error: {:?}", e);
+                            }
+                            send_batches_time.stop();
+
+                            stats.apply_sender_stakes_time = stats
+                                .apply_sender_stakes_time
+                                .saturating_add(apply_sender_stakes_time.as_us());
+                            stats.send_batches_time = stats
+                                .send_batches_time
+                                .saturating_add(send_batches_time.as_us());
+                            stats.receive_batches_time = stats
+                                .receive_batches_time
+                                .saturating_add(recv_duration.as_nanos() as u64);
+                            stats.total_batches =
+                                stats.total_batches.saturating_add(num_batches as u64);
+                            stats.total_packets =
+                                stats.total_packets.saturating_add(num_packets as u64);
+                        }
+                        Err(e) => match e {
+                            StreamerError::RecvTimeout(RecvTimeoutError::Disconnected) => break,
+                            StreamerError::RecvTimeout(RecvTimeoutError::Timeout) => (),
+                            _ => error!("error: {:?}", e),
+                        },
+                    }
+
+                    stats.report();
+                }
+            })
+            .unwrap();
+        Self { thread_hdl }
+    }
+
+    fn try_refresh_ip_to_stake(
+        last_stakes: &mut Instant,
+        ip_to_stake: &mut HashMap<IpAddr, u64>,
+        bank_forks: Arc<RwLock<BankForks>>,
+        cluster_info: Arc<ClusterInfo>,
+    ) {
+        if last_stakes.elapsed() > IP_TO_STAKE_REFRESH_DURATION {
+            let root_bank = bank_forks.read().unwrap().root_bank();
+            let staked_nodes = root_bank.staked_nodes();
+            *ip_to_stake = cluster_info
+                .tvu_peers()
+                .into_iter()
+                .filter_map(|node| {
+                    let stake = staked_nodes.get(&node.id)?;
+                    Some((node.tvu.ip(), *stake))
+                })
+                .collect();
+            *last_stakes = Instant::now();
+        }
+    }
+
+    fn apply_sender_stakes(batches: &mut [PacketBatch], ip_to_stake: &HashMap<IpAddr, u64>) {
+        PAR_THREAD_POOL.with(|thread_pool| {
+            thread_pool.borrow().install(|| {
+                batches
+                    .into_par_iter()
+                    .flat_map(|batch| batch.packets.par_iter_mut())
+                    .for_each(|packet| {
+                        packet.meta.sender_stake =
+                            *ip_to_stake.get(&packet.meta.addr().ip()).unwrap_or(&0);
+                    });
+            })
+        });
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.thread_hdl.join()
+    }
+}

core/src/ledger_metric_report_service.rs

@@ -0,0 +1,44 @@
+//! The `ledger_metric_report_service` periodically reports ledger store metrics.
+
+use {
+    solana_ledger::blockstore::Blockstore,
+    std::{
+        string::ToString,
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc,
+        },
+        thread::{self, Builder, JoinHandle},
+        time::Duration,
+    },
+};
+
+// Determines how often we report blockstore metrics.
+const BLOCKSTORE_METRICS_REPORT_PERIOD_MILLIS: u64 = 10000;
+
+pub struct LedgerMetricReportService {
+    t_cf_metric: JoinHandle<()>,
+}
+
+impl LedgerMetricReportService {
+    pub fn new(blockstore: Arc<Blockstore>, exit: &Arc<AtomicBool>) -> Self {
+        let exit_signal = exit.clone();
+        let t_cf_metric = Builder::new()
+            .name("metric_report_rocksdb_cf_metrics".to_string())
+            .spawn(move || loop {
+                if exit_signal.load(Ordering::Relaxed) {
+                    break;
+                }
+                thread::sleep(Duration::from_millis(
+                    BLOCKSTORE_METRICS_REPORT_PERIOD_MILLIS,
+                ));
+                blockstore.submit_rocksdb_cf_metrics_for_all_cfs();
+            })
+            .unwrap();
+        Self { t_cf_metric }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.t_cf_metric.join()
+    }
+}

core/src/lib.rs

@@ -24,6 +24,7 @@ pub mod cost_update_service;
 pub mod drop_bank_service;
 pub mod duplicate_repair_status;
 pub mod fetch_stage;
+pub mod find_packet_sender_stake_stage;
 pub mod fork_choice;
 pub mod gen_keys;
 pub mod heaviest_subtree_fork_choice;
@@ -31,9 +32,11 @@ pub mod latest_validator_votes_for_frozen_banks;
 pub mod leader_slot_banking_stage_metrics;
 pub mod leader_slot_banking_stage_timing_metrics;
 pub mod ledger_cleanup_service;
+pub mod ledger_metric_report_service;
 pub mod optimistic_confirmation_verifier;
 pub mod outstanding_requests;
 pub mod packet_hasher;
+pub mod packet_threshold;
 pub mod progress_map;
 pub mod qos_service;
 pub mod repair_generic_traversal;
@@ -62,6 +65,7 @@ pub mod tpu;
 pub mod tree_diff;
 pub mod tvu;
 pub mod unfrozen_gossip_verified_vote_hashes;
+pub mod unprocessed_packet_batches;
 pub mod validator;
 pub mod verified_vote_packets;
 pub mod vote_simulator;

core/src/packet_threshold.rs

@@ -0,0 +1,84 @@
+use std::time::Duration;
+
+enum PacketThresholdUpdate {
+    Increase,
+    Decrease,
+}
+
+impl PacketThresholdUpdate {
+    const PERCENTAGE: usize = 90;
+
+    fn calculate(&self, current: usize) -> usize {
+        match *self {
+            PacketThresholdUpdate::Increase => {
+                current.saturating_mul(100).saturating_div(Self::PERCENTAGE)
+            }
+            PacketThresholdUpdate::Decrease => {
+                current.saturating_mul(Self::PERCENTAGE).saturating_div(100)
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct DynamicPacketToProcessThreshold {
+    max_packets: usize,
+}
+
+impl Default for DynamicPacketToProcessThreshold {
+    fn default() -> Self {
+        Self {
+            max_packets: Self::DEFAULT_MAX_PACKETS,
+        }
+    }
+}
+
+impl DynamicPacketToProcessThreshold {
+    const DEFAULT_MAX_PACKETS: usize = 1024;
+    const TIME_THRESHOLD: Duration = Duration::from_secs(1);
+
+    pub fn update(&mut self, total_packets: usize, compute_time: Duration) {
+        if total_packets >= self.max_packets {
+            let threshold_update = if compute_time > Self::TIME_THRESHOLD {
+                PacketThresholdUpdate::Decrease
+            } else {
+                PacketThresholdUpdate::Increase
+            };
+            self.max_packets = threshold_update.calculate(self.max_packets);
+        }
+    }
+
+    pub fn should_drop(&self, total: usize) -> bool {
+        total >= self.max_packets
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use {super::DynamicPacketToProcessThreshold, std::time::Duration};
+
+    #[test]
+    fn test_dynamic_packet_threshold() {
+        let mut threshold = DynamicPacketToProcessThreshold::default();
+        assert_eq!(
+            threshold.max_packets,
+            DynamicPacketToProcessThreshold::DEFAULT_MAX_PACKETS
+        );
+
+        assert!(!threshold.should_drop(10));
+        assert!(threshold.should_drop(2000));
+
+        let old = threshold.max_packets;
+
+        // Increase
+        let total = 2000;
+        let compute_time = Duration::from_millis(500);
+        threshold.update(total, compute_time);
+        assert!(threshold.max_packets > old);
+
+        // Decrease
+        let compute_time = Duration::from_millis(2000);
+        threshold.update(total, compute_time);
+        assert_eq!(threshold.max_packets, old - 1); // due to rounding error, there is a difference of 1
+    }
+}

core/src/progress_map.rs

@@ -339,7 +339,7 @@ pub struct RetransmitInfo {
 impl RetransmitInfo {
     pub fn reached_retransmit_threshold(&self) -> bool {
         let backoff = std::cmp::min(self.retry_iteration, RETRANSMIT_BACKOFF_CAP);
-        let backoff_duration_ms = 2_u64.pow(backoff) * RETRANSMIT_BASE_DELAY_MS;
+        let backoff_duration_ms = (1_u64 << backoff) * RETRANSMIT_BASE_DELAY_MS;
         self.retry_time
             .map(|time| time.elapsed().as_millis() > backoff_duration_ms.into())
             .unwrap_or(true)

core/src/qos_service.rs

@@ -3,7 +3,7 @@
 //! how transactions are included in blocks, and optimize those blocks.
 //!
 use {
-    crate::banking_stage::BatchedTransactionCostDetails,
+    crate::banking_stage::BatchedTransactionDetails,
     crossbeam_channel::{unbounded, Receiver, Sender},
     solana_measure::measure::Measure,
     solana_runtime::{
@@ -68,8 +68,8 @@ impl QosService {
         let running_flag = Arc::new(AtomicBool::new(true));
         let metrics = Arc::new(QosServiceMetrics::new(id));
 
-        let running_flag_clone = running_flag.clone();
-        let metrics_clone = metrics.clone();
+        let running_flag_clone = Arc::clone(&running_flag);
+        let metrics_clone = Arc::clone(&metrics);
         let reporting_thread = Some(
             Builder::new()
                 .name("solana-qos-service-metrics-repoting".to_string())
@@ -109,9 +109,11 @@ impl QosService {
             .collect();
         compute_cost_time.stop();
         self.metrics
+            .stats
             .compute_cost_time
             .fetch_add(compute_cost_time.as_us(), Ordering::Relaxed);
         self.metrics
+            .stats
             .compute_cost_count
             .fetch_add(txs_costs.len() as u64, Ordering::Relaxed);
         txs_costs
@@ -134,7 +136,7 @@ impl QosService {
             .map(|(tx, cost)| match cost_tracker.try_add(tx, cost) {
                 Ok(current_block_cost) => {
                     debug!("slot {:?}, transaction {:?}, cost {:?}, fit into current block, current block cost {}", bank.slot(), tx, cost, current_block_cost);
-                    self.metrics.selected_txs_count.fetch_add(1, Ordering::Relaxed);
+                    self.metrics.stats.selected_txs_count.fetch_add(1, Ordering::Relaxed);
                     num_included += 1;
                     Ok(())
                 },
@@ -142,20 +144,19 @@ impl QosService {
                     debug!("slot {:?}, transaction {:?}, cost {:?}, not fit into current block, '{:?}'", bank.slot(), tx, cost, e);
                     match e {
                         CostTrackerError::WouldExceedBlockMaxLimit => {
-                            self.metrics.retried_txs_per_block_limit_count.fetch_add(1, Ordering::Relaxed);
                             Err(TransactionError::WouldExceedMaxBlockCostLimit)
                         }
                         CostTrackerError::WouldExceedVoteMaxLimit => {
-                            self.metrics.retried_txs_per_vote_limit_count.fetch_add(1, Ordering::Relaxed);
                             Err(TransactionError::WouldExceedMaxVoteCostLimit)
                         }
                         CostTrackerError::WouldExceedAccountMaxLimit => {
-                            self.metrics.retried_txs_per_account_limit_count.fetch_add(1, Ordering::Relaxed);
                             Err(TransactionError::WouldExceedMaxAccountCostLimit)
                         }
-                        CostTrackerError::WouldExceedAccountDataMaxLimit => {
-                            self.metrics.retried_txs_per_account_data_limit_count.fetch_add(1, Ordering::Relaxed);
-                            Err(TransactionError::WouldExceedMaxAccountDataCostLimit)
+                        CostTrackerError::WouldExceedAccountDataBlockLimit => {
+                            Err(TransactionError::WouldExceedAccountDataBlockLimit)
+                        }
+                        CostTrackerError::WouldExceedAccountDataTotalLimit => {
+                            Err(TransactionError::WouldExceedAccountDataTotalLimit)
                         }
                     }
                 }
@@ -163,6 +164,7 @@ impl QosService {
             .collect();
         cost_tracking_time.stop();
         self.metrics
+            .stats
             .cost_tracking_time
             .fetch_add(cost_tracking_time.as_us(), Ordering::Relaxed);
         (select_results, num_included)
@@ -177,30 +179,82 @@ impl QosService {
 
     pub fn accumulate_estimated_transaction_costs(
         &self,
-        cost_details: &BatchedTransactionCostDetails,
+        batched_transaction_details: &BatchedTransactionDetails,
     ) {
+        self.metrics.stats.estimated_signature_cu.fetch_add(
+            batched_transaction_details.costs.batched_signature_cost,
+            Ordering::Relaxed,
+        );
+        self.metrics.stats.estimated_write_lock_cu.fetch_add(
+            batched_transaction_details.costs.batched_write_lock_cost,
+            Ordering::Relaxed,
+        );
+        self.metrics.stats.estimated_data_bytes_cu.fetch_add(
+            batched_transaction_details.costs.batched_data_bytes_cost,
+            Ordering::Relaxed,
+        );
+        self.metrics.stats.estimated_execute_cu.fetch_add(
+            batched_transaction_details.costs.batched_execute_cost,
+            Ordering::Relaxed,
+        );
+
         self.metrics
-            .estimated_signature_cu
-            .fetch_add(cost_details.batched_signature_cost, Ordering::Relaxed);
+            .errors
+            .retried_txs_per_block_limit_count
+            .fetch_add(
+                batched_transaction_details
+                    .errors
+                    .batched_retried_txs_per_block_limit_count,
+                Ordering::Relaxed,
+            );
         self.metrics
-            .estimated_write_lock_cu
-            .fetch_add(cost_details.batched_write_lock_cost, Ordering::Relaxed);
+            .errors
+            .retried_txs_per_vote_limit_count
+            .fetch_add(
+                batched_transaction_details
+                    .errors
+                    .batched_retried_txs_per_vote_limit_count,
+                Ordering::Relaxed,
+            );
         self.metrics
-            .estimated_data_bytes_cu
-            .fetch_add(cost_details.batched_data_bytes_cost, Ordering::Relaxed);
+            .errors
+            .retried_txs_per_account_limit_count
+            .fetch_add(
+                batched_transaction_details
+                    .errors
+                    .batched_retried_txs_per_account_limit_count,
+                Ordering::Relaxed,
+            );
         self.metrics
-            .estimated_execute_cu
-            .fetch_add(cost_details.batched_execute_cost, Ordering::Relaxed);
+            .errors
+            .retried_txs_per_account_data_block_limit_count
+            .fetch_add(
+                batched_transaction_details
+                    .errors
+                    .batched_retried_txs_per_account_data_block_limit_count,
+                Ordering::Relaxed,
+            );
+        self.metrics
+            .errors
+            .dropped_txs_per_account_data_total_limit_count
+            .fetch_add(
+                batched_transaction_details
+                    .errors
+                    .batched_dropped_txs_per_account_data_total_limit_count,
+                Ordering::Relaxed,
+            );
     }
 
     pub fn accumulate_actual_execute_cu(&self, units: u64) {
         self.metrics
+            .stats
             .actual_execute_cu
             .fetch_add(units, Ordering::Relaxed);
     }
 
     pub fn accumulate_actual_execute_time(&self, micro_sec: u64) {
         self.metrics
+            .stats
             .actual_execute_time_us
             .fetch_add(micro_sec, Ordering::Relaxed);
     }
@@ -223,63 +277,77 @@ impl QosService {
     }
 }
 
-#[derive(Default)]
+#[derive(Debug, Default)]
 struct QosServiceMetrics {
-    // banking_stage creates one QosService instance per working threads, that is uniquely
-    // identified by id. This field allows to categorize metrics for gossip votes, TPU votes
-    // and other transactions.
+    /// banking_stage creates one QosService instance per working threads, that is uniquely
+    /// identified by id. This field allows to categorize metrics for gossip votes, TPU votes
+    /// and other transactions.
     id: u32,
 
-    // aggregate metrics per slot
+    /// aggregate metrics per slot
     slot: AtomicU64,
 
-    // accumulated time in micro-sec spent in computing transaction cost. It is the main performance
-    // overhead introduced by cost_model
+    stats: QosServiceMetricsStats,
+    errors: QosServiceMetricsErrors,
+}
+
+#[derive(Debug, Default)]
+struct QosServiceMetricsStats {
+    /// accumulated time in micro-sec spent in computing transaction cost. It is the main performance
+    /// overhead introduced by cost_model
     compute_cost_time: AtomicU64,
 
-    // total nummber of transactions in the reporting period to be computed for theit cost. It is
-    // usually the number of sanitized transactions leader receives.
+    /// total nummber of transactions in the reporting period to be computed for theit cost. It is
+    /// usually the number of sanitized transactions leader receives.
     compute_cost_count: AtomicU64,
 
-    // acumulated time in micro-sec spent in tracking each bank's cost. It is the second part of
-    // overhead introduced
+    /// acumulated time in micro-sec spent in tracking each bank's cost. It is the second part of
+    /// overhead introduced
     cost_tracking_time: AtomicU64,
 
-    // number of transactions to be included in blocks
+    /// number of transactions to be included in blocks
     selected_txs_count: AtomicU64,
 
-    // number of transactions to be queued for retry due to its potential to breach block limit
-    retried_txs_per_block_limit_count: AtomicU64,
-
-    // number of transactions to be queued for retry due to its potential to breach vote limit
-    retried_txs_per_vote_limit_count: AtomicU64,
-
-    // number of transactions to be queued for retry due to its potential to breach writable
-    // account limit
-    retried_txs_per_account_limit_count: AtomicU64,
-
-    // number of transactions to be queued for retry due to its account data limits
-    retried_txs_per_account_data_limit_count: AtomicU64,
-
-    // accumulated estimated signature Compute Unites to be packed into block
+    /// accumulated estimated signature Compute Unites to be packed into block
     estimated_signature_cu: AtomicU64,
 
-    // accumulated estimated write locks Compute Units to be packed into block
+    /// accumulated estimated write locks Compute Units to be packed into block
     estimated_write_lock_cu: AtomicU64,
 
-    // accumulated estimated instructino data Compute Units to be packed into block
+    /// accumulated estimated instructino data Compute Units to be packed into block
     estimated_data_bytes_cu: AtomicU64,
 
-    // accumulated estimated program Compute Units to be packed into block
+    /// accumulated estimated program Compute Units to be packed into block
     estimated_execute_cu: AtomicU64,
 
-    // accumulated actual program Compute Units that have been packed into block
+    /// accumulated actual program Compute Units that have been packed into block
     actual_execute_cu: AtomicU64,
 
-    // accumulated actual program execute micro-sec that have been packed into block
+    /// accumulated actual program execute micro-sec that have been packed into block
     actual_execute_time_us: AtomicU64,
 }
 
+#[derive(Debug, Default)]
+struct QosServiceMetricsErrors {
+    /// number of transactions to be queued for retry due to their potential to breach block limit
+    retried_txs_per_block_limit_count: AtomicU64,
+
+    /// number of transactions to be queued for retry due to their potential to breach vote limit
+    retried_txs_per_vote_limit_count: AtomicU64,
+
+    /// number of transactions to be queued for retry due to their potential to breach writable
+    /// account limit
+    retried_txs_per_account_limit_count: AtomicU64,
+
+    /// number of transactions to be queued for retry due to their potential to breach account data
+    /// block limits
+    retried_txs_per_account_data_block_limit_count: AtomicU64,
+
+    /// number of transactions to be dropped due to their potential to breach account data total
+    /// limits
+    dropped_txs_per_account_data_total_limit_count: AtomicU64,
+}
+
 impl QosServiceMetrics {
     pub fn new(id: u32) -> Self {
         QosServiceMetrics {
@@ -296,76 +364,96 @@ impl QosServiceMetrics {
                 ("bank_slot", bank_slot as i64, i64),
                 (
                     "compute_cost_time",
-                    self.compute_cost_time.swap(0, Ordering::Relaxed) as i64,
+                    self.stats.compute_cost_time.swap(0, Ordering::Relaxed) as i64,
                     i64
                 ),
                 (
                     "compute_cost_count",
-                    self.compute_cost_count.swap(0, Ordering::Relaxed) as i64,
+                    self.stats.compute_cost_count.swap(0, Ordering::Relaxed) as i64,
                     i64
                 ),
                 (
                     "cost_tracking_time",
-                    self.cost_tracking_time.swap(0, Ordering::Relaxed) as i64,
+                    self.stats.cost_tracking_time.swap(0, Ordering::Relaxed) as i64,
                     i64
                 ),
                 (
                     "selected_txs_count",
-                    self.selected_txs_count.swap(0, Ordering::Relaxed) as i64,
+                    self.stats.selected_txs_count.swap(0, Ordering::Relaxed) as i64,
                     i64
                 ),
+                (
+                    "estimated_signature_cu",
+                    self.stats.estimated_signature_cu.swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+                (
+                    "estimated_write_lock_cu",
+                    self.stats
+                        .estimated_write_lock_cu
+                        .swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+                (
+                    "estimated_data_bytes_cu",
+                    self.stats
+                        .estimated_data_bytes_cu
+                        .swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+                (
+                    "estimated_execute_cu",
+                    self.stats.estimated_execute_cu.swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+                (
+                    "actual_execute_cu",
+                    self.stats.actual_execute_cu.swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+                (
+                    "actual_execute_time_us",
+                    self.stats.actual_execute_time_us.swap(0, Ordering::Relaxed) as i64,
+                    i64
+                ),
+            );
+            datapoint_info!(
+                "qos-service-errors",
+                ("id", self.id as i64, i64),
+                ("bank_slot", bank_slot as i64, i64),
                 (
                     "retried_txs_per_block_limit_count",
-                    self.retried_txs_per_block_limit_count
+                    self.errors
+                        .retried_txs_per_block_limit_count
                         .swap(0, Ordering::Relaxed) as i64,
                     i64
                 ),
                 (
                     "retried_txs_per_vote_limit_count",
-                    self.retried_txs_per_vote_limit_count
+                    self.errors
+                        .retried_txs_per_vote_limit_count
                         .swap(0, Ordering::Relaxed) as i64,
                     i64
                 ),
                 (
                     "retried_txs_per_account_limit_count",
-                    self.retried_txs_per_account_limit_count
+                    self.errors
+                        .retried_txs_per_account_limit_count
                         .swap(0, Ordering::Relaxed) as i64,
                     i64
                 ),
                 (
-                    "retried_txs_per_account_data_limit_count",
-                    self.retried_txs_per_account_data_limit_count
+                    "retried_txs_per_account_data_block_limit_count",
+                    self.errors
+                        .retried_txs_per_account_data_block_limit_count
                         .swap(0, Ordering::Relaxed) as i64,
                     i64
                 ),
                 (
-                    "estimated_signature_cu",
-                    self.estimated_signature_cu.swap(0, Ordering::Relaxed) as i64,
-                    i64
-                ),
-                (
-                    "estimated_write_lock_cu",
-                    self.estimated_write_lock_cu.swap(0, Ordering::Relaxed) as i64,
-                    i64
-                ),
-                (
-                    "estimated_data_bytes_cu",
-                    self.estimated_data_bytes_cu.swap(0, Ordering::Relaxed) as i64,
-                    i64
-                ),
-                (
-                    "estimated_execute_cu",
-                    self.estimated_execute_cu.swap(0, Ordering::Relaxed) as i64,
-                    i64
-                ),
-                (
-                    "actual_execute_cu",
-                    self.actual_execute_cu.swap(0, Ordering::Relaxed) as i64,
-                    i64
-                ),
-                (
-                    "actual_execute_time_us",
-                    self.actual_execute_time_us.swap(0, Ordering::Relaxed) as i64,
+                    "dropped_txs_per_account_data_total_limit_count",
+                    self.errors
+                        .dropped_txs_per_account_data_total_limit_count
+                        .swap(0, Ordering::Relaxed) as i64,
                     i64
                 ),
             );

core/src/replay_stage.rs

@@ -27,9 +27,9 @@ use {
         window_service::DuplicateSlotReceiver,
     },
     crossbeam_channel::{Receiver, RecvTimeoutError, Sender},
-    solana_accountsdb_plugin_manager::block_metadata_notifier_interface::BlockMetadataNotifierLock,
     solana_client::rpc_response::SlotUpdate,
     solana_entry::entry::VerifyRecyclers,
+    solana_geyser_plugin_manager::block_metadata_notifier_interface::BlockMetadataNotifierLock,
     solana_gossip::cluster_info::ClusterInfo,
     solana_ledger::{
         block_error::BlockError,
@@ -735,11 +735,16 @@ impl ReplayStage {
                                         restored_tower.adjust_lockouts_after_replay(root_bank.slot(), &slot_history)
                                     }).
                                     unwrap_or_else(|err| {
-                                        // It's a fatal error if the tower is not present.  This is
-                                        // necessary to prevent the validator from violating
-                                        // lockouts for its new identity
-                                        error!("Failed to load tower for {}: {}", my_pubkey, err);
-                                        std::process::exit(1);
+                                        if err.is_file_missing() {
+                                            Tower::new_from_bankforks(
+                                                &bank_forks.read().unwrap(),
+                                                &my_pubkey,
+                                                &vote_account,
+                                            )
+                                        } else {
+                                            error!("Failed to load tower for {}: {}", my_pubkey, err);
+                                            std::process::exit(1);
+                                        }
                                     });
 
                                 // Ensure the validator can land votes with the new identity before
@@ -1729,7 +1734,7 @@ impl ReplayStage {
         replay_timing: &mut ReplayTiming,
         voting_sender: &Sender<VoteOp>,
         epoch_slots_frozen_slots: &mut EpochSlotsFrozenSlots,
-        bank_drop_sender: &Sender<Vec<Arc<Bank>>>,
+        drop_bank_sender: &Sender<Vec<Arc<Bank>>>,
         wait_to_vote_slot: Option<Slot>,
     ) {
         if bank.is_empty() {
@@ -1781,7 +1786,7 @@ impl ReplayStage {
                 has_new_vote_been_rooted,
                 vote_signatures,
                 epoch_slots_frozen_slots,
-                bank_drop_sender,
+                drop_bank_sender,
             );
             rpc_subscriptions.notify_roots(rooted_slots);
             if let Some(sender) = bank_notification_sender {
@@ -2919,14 +2924,14 @@ impl ReplayStage {
         has_new_vote_been_rooted: &mut bool,
         voted_signatures: &mut Vec<Signature>,
         epoch_slots_frozen_slots: &mut EpochSlotsFrozenSlots,
-        bank_drop_sender: &Sender<Vec<Arc<Bank>>>,
+        drop_bank_sender: &Sender<Vec<Arc<Bank>>>,
     ) {
         let removed_banks = bank_forks.write().unwrap().set_root(
             new_root,
             accounts_background_request_sender,
             highest_confirmed_root,
         );
-        bank_drop_sender
+        drop_bank_sender
             .send(removed_banks)
             .unwrap_or_else(|err| warn!("bank drop failed: {:?}", err));
 
@@ -3129,7 +3134,7 @@ pub mod tests {
         },
         solana_rpc::{
             optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank,
-            rpc::create_test_transactions_and_populate_blockstore,
+            rpc::{create_test_transaction_entries, populate_blockstore_for_tests},
         },
         solana_runtime::{
             accounts_background_service::AbsRequestSender,
@@ -3437,7 +3442,7 @@ pub mod tests {
             .into_iter()
             .map(|slot| (slot, Hash::default()))
             .collect();
-        let (bank_drop_sender, _bank_drop_receiver) = unbounded();
+        let (drop_bank_sender, _drop_bank_receiver) = unbounded();
         ReplayStage::handle_new_root(
             root,
             &bank_forks,
@@ -3451,7 +3456,7 @@ pub mod tests {
             &mut true,
             &mut Vec::new(),
             &mut epoch_slots_frozen_slots,
-            &bank_drop_sender,
+            &drop_bank_sender,
         );
         assert_eq!(bank_forks.read().unwrap().root(), root);
         assert_eq!(progress.len(), 1);
@@ -3518,7 +3523,7 @@ pub mod tests {
         for i in 0..=root {
             progress.insert(i, ForkProgress::new(Hash::default(), None, None, 0, 0));
         }
-        let (bank_drop_sender, _bank_drop_receiver) = unbounded();
+        let (drop_bank_sender, _drop_bank_receiver) = unbounded();
         ReplayStage::handle_new_root(
             root,
             &bank_forks,
@@ -3532,7 +3537,7 @@ pub mod tests {
             &mut true,
             &mut Vec::new(),
             &mut EpochSlotsFrozenSlots::default(),
-            &bank_drop_sender,
+            &drop_bank_sender,
         );
         assert_eq!(bank_forks.read().unwrap().root(), root);
         assert!(bank_forks.read().unwrap().get(confirmed_root).is_some());
@@ -3998,15 +4003,18 @@ pub mod tests {
             let bank1 = Arc::new(Bank::new_from_parent(&bank0, &Pubkey::default(), 1));
             let slot = bank1.slot();
 
-            let mut test_signatures_iter = create_test_transactions_and_populate_blockstore(
+            let (entries, test_signatures) = create_test_transaction_entries(
                 vec![&mint_keypair, &keypair1, &keypair2, &keypair3],
-                bank0.slot(),
+                bank1.clone(),
+            );
+            populate_blockstore_for_tests(
+                entries,
                 bank1,
                 blockstore.clone(),
                 Arc::new(AtomicU64::default()),
-            )
-            .into_iter();
+            );
 
+            let mut test_signatures_iter = test_signatures.into_iter();
             let confirmed_block = blockstore.get_rooted_block(slot, false).unwrap();
             let actual_tx_results: Vec<_> = confirmed_block
                 .transactions

core/src/retransmit_stage.rs

@@ -416,7 +416,7 @@ pub fn retransmitter(
         .unwrap()
 }
 
-pub(crate) struct RetransmitStage {
+pub struct RetransmitStage {
     retransmit_thread_handle: JoinHandle<()>,
     window_service: WindowService,
     cluster_slots_service: ClusterSlotsService,
@@ -529,7 +529,7 @@ mod tests {
         super::*,
         solana_gossip::contact_info::ContactInfo,
         solana_ledger::{
-            blockstore_processor::{process_blockstore, ProcessOptions},
+            blockstore_processor::{test_process_blockstore, ProcessOptions},
             create_new_tmp_ledger,
             genesis_utils::{create_genesis_config, GenesisConfigInfo},
         },
@@ -550,19 +550,9 @@ mod tests {
             full_leader_cache: true,
             ..ProcessOptions::default()
         };
-        let (accounts_package_sender, _) = unbounded();
-        let (bank_forks, cached_leader_schedule, _) = process_blockstore(
-            &genesis_config,
-            &blockstore,
-            Vec::new(),
-            opts,
-            None,
-            None,
-            accounts_package_sender,
-            None,
-        )
-        .unwrap();
-        let leader_schedule_cache = Arc::new(cached_leader_schedule);
+        let (bank_forks, leader_schedule_cache) =
+            test_process_blockstore(&genesis_config, &blockstore, opts);
+        let leader_schedule_cache = Arc::new(leader_schedule_cache);
         let bank_forks = Arc::new(RwLock::new(bank_forks));
 
         let mut me = ContactInfo::new_localhost(&solana_sdk::pubkey::new_rand(), 0);

core/src/serve_repair.rs

@@ -2,6 +2,7 @@ use {
     crate::{
         cluster_slots::ClusterSlots,
         duplicate_repair_status::ANCESTOR_HASH_REPAIR_SAMPLE_SIZE,
+        packet_threshold::DynamicPacketToProcessThreshold,
         repair_response,
         repair_service::{OutstandingShredRepairs, RepairStats},
         request_response::RequestResponse,
@@ -23,7 +24,6 @@ use {
         blockstore::Blockstore,
         shred::{Nonce, Shred, SIZE_OF_NONCE},
     },
-    solana_measure::measure::Measure,
     solana_metrics::inc_new_counter_debug,
     solana_perf::packet::{limited_deserialize, PacketBatch, PacketBatchRecycler},
     solana_sdk::{
@@ -322,7 +322,7 @@ impl ServeRepair {
         requests_receiver: &PacketBatchReceiver,
         response_sender: &PacketBatchSender,
         stats: &mut ServeRepairStats,
-        max_packets: &mut usize,
+        packet_threshold: &mut DynamicPacketToProcessThreshold,
     ) -> Result<()> {
         //TODO cache connections
         let timeout = Duration::new(1, 0);
@@ -332,29 +332,21 @@ impl ServeRepair {
         let mut dropped_packets = 0;
         while let Ok(more) = requests_receiver.try_recv() {
             total_packets += more.packets.len();
-            if total_packets < *max_packets {
-                // Drop the rest in the channel in case of dos
-                reqs_v.push(more);
-            } else {
+            if packet_threshold.should_drop(total_packets) {
                 dropped_packets += more.packets.len();
+            } else {
+                reqs_v.push(more);
             }
         }
 
         stats.dropped_packets += dropped_packets;
         stats.total_packets += total_packets;
 
-        let mut time = Measure::start("repair::handle_packets");
+        let timer = Instant::now();
         for reqs in reqs_v {
             Self::handle_packets(obj, recycler, blockstore, reqs, response_sender, stats);
         }
-        time.stop();
-        if total_packets >= *max_packets {
-            if time.as_ms() > 1000 {
-                *max_packets = (*max_packets * 9) / 10;
-            } else {
-                *max_packets = (*max_packets * 10) / 9;
-            }
-        }
+        packet_threshold.update(total_packets, timer.elapsed());
         Ok(())
     }
 
@@ -403,7 +395,7 @@ impl ServeRepair {
             .spawn(move || {
                 let mut last_print = Instant::now();
                 let mut stats = ServeRepairStats::default();
-                let mut max_packets = 1024;
+                let mut packet_threshold = DynamicPacketToProcessThreshold::default();
                 loop {
                     let result = Self::run_listen(
                         &me,
@@ -412,7 +404,7 @@ impl ServeRepair {
                         &requests_receiver,
                         &response_sender,
                         &mut stats,
-                        &mut max_packets,
+                        &mut packet_threshold,
                     );
                     match result {
                         Err(Error::RecvTimeout(_)) | Ok(_) => {}

core/src/shred_fetch_stage.rs

@@ -2,17 +2,17 @@
 
 use {
     crate::packet_hasher::PacketHasher,
-    crossbeam_channel::unbounded,
+    crossbeam_channel::{unbounded, Sender},
     lru::LruCache,
     solana_ledger::shred::{get_shred_slot_index_type, ShredFetchStats},
     solana_perf::{
         cuda_runtime::PinnedVec,
-        packet::{Packet, PacketBatchRecycler, PacketFlags},
+        packet::{Packet, PacketBatch, PacketBatchRecycler, PacketFlags},
         recycler::Recycler,
     },
     solana_runtime::bank_forks::BankForks,
     solana_sdk::clock::{Slot, DEFAULT_MS_PER_SLOT},
-    solana_streamer::streamer::{self, PacketBatchReceiver, PacketBatchSender},
+    solana_streamer::streamer::{self, PacketBatchReceiver},
     std::{
         net::UdpSocket,
         sync::{atomic::AtomicBool, Arc, RwLock},
@@ -65,7 +65,7 @@ impl ShredFetchStage {
     // updates packets received on a channel and sends them on another channel
     fn modify_packets<F>(
         recvr: PacketBatchReceiver,
-        sendr: PacketBatchSender,
+        sendr: Sender<Vec<PacketBatch>>,
         bank_forks: Option<Arc<RwLock<BankForks>>>,
         name: &'static str,
         modify: F,
@@ -125,7 +125,7 @@ impl ShredFetchStage {
                 stats = ShredFetchStats::default();
                 last_stats = Instant::now();
             }
-            if sendr.send(packet_batch).is_err() {
+            if sendr.send(vec![packet_batch]).is_err() {
                 break;
             }
         }
@@ -134,7 +134,7 @@ impl ShredFetchStage {
     fn packet_modifier<F>(
         sockets: Vec<Arc<UdpSocket>>,
         exit: &Arc<AtomicBool>,
-        sender: PacketBatchSender,
+        sender: Sender<Vec<PacketBatch>>,
         recycler: Recycler<PinnedVec<Packet>>,
         bank_forks: Option<Arc<RwLock<BankForks>>>,
         name: &'static str,
@@ -170,7 +170,7 @@ impl ShredFetchStage {
         sockets: Vec<Arc<UdpSocket>>,
         forward_sockets: Vec<Arc<UdpSocket>>,
         repair_socket: Arc<UdpSocket>,
-        sender: &PacketBatchSender,
+        sender: &Sender<Vec<PacketBatch>>,
         bank_forks: Option<Arc<RwLock<BankForks>>>,
         exit: &Arc<AtomicBool>,
     ) -> Self {

core/src/sigverify_stage.rs

@@ -6,9 +6,9 @@
 //! if perf-libs are available
 
 use {
-    crate::sigverify,
+    crate::{find_packet_sender_stake_stage, sigverify},
     core::time::Duration,
-    crossbeam_channel::{Receiver, RecvTimeoutError, SendError, Sender},
+    crossbeam_channel::{RecvTimeoutError, SendError, Sender},
     itertools::Itertools,
     solana_measure::measure::Measure,
     solana_perf::{
@@ -16,7 +16,7 @@ use {
         sigverify::{count_valid_packets, shrink_batches, Deduper},
     },
     solana_sdk::timing,
-    solana_streamer::streamer::{self, PacketBatchReceiver, StreamerError},
+    solana_streamer::streamer::{self, StreamerError},
     std::{
         thread::{self, Builder, JoinHandle},
         time::Instant,
@@ -192,7 +192,7 @@ impl SigVerifier for DisabledSigVerifier {
 impl SigVerifyStage {
     #[allow(clippy::new_ret_no_self)]
     pub fn new<T: SigVerifier + 'static + Send + Clone>(
-        packet_receiver: Receiver<PacketBatch>,
+        packet_receiver: find_packet_sender_stake_stage::FindPacketSenderStakeReceiver,
         verified_sender: Sender<Vec<PacketBatch>>,
         verifier: T,
     ) -> Self {
@@ -227,12 +227,12 @@ impl SigVerifyStage {
 
     fn verifier<T: SigVerifier>(
         deduper: &Deduper,
-        recvr: &PacketBatchReceiver,
+        recvr: &find_packet_sender_stake_stage::FindPacketSenderStakeReceiver,
         sendr: &Sender<Vec<PacketBatch>>,
         verifier: &T,
         stats: &mut SigVerifierStats,
     ) -> Result<()> {
-        let (mut batches, num_packets, recv_duration) = streamer::recv_packet_batches(recvr)?;
+        let (mut batches, num_packets, recv_duration) = streamer::recv_vec_packet_batches(recvr)?;
 
         let batches_len = batches.len();
         debug!(
@@ -312,7 +312,7 @@ impl SigVerifyStage {
     }
 
     fn verifier_service<T: SigVerifier + 'static + Send + Clone>(
-        packet_receiver: PacketBatchReceiver,
+        packet_receiver: find_packet_sender_stake_stage::FindPacketSenderStakeReceiver,
         verified_sender: Sender<Vec<PacketBatch>>,
         verifier: &T,
     ) -> JoinHandle<()> {
@@ -358,7 +358,7 @@ impl SigVerifyStage {
     }
 
     fn verifier_services<T: SigVerifier + 'static + Send + Clone>(
-        packet_receiver: PacketBatchReceiver,
+        packet_receiver: find_packet_sender_stake_stage::FindPacketSenderStakeReceiver,
         verified_sender: Sender<Vec<PacketBatch>>,
         verifier: T,
     ) -> JoinHandle<()> {
@@ -445,7 +445,7 @@ mod tests {
         for _ in 0..batches.len() {
             if let Some(batch) = batches.pop() {
                 sent_len += batch.packets.len();
-                packet_s.send(batch).unwrap();
+                packet_s.send(vec![batch]).unwrap();
             }
         }
         let mut received = 0;

core/src/snapshot_packager_service.rs

@@ -219,7 +219,7 @@ mod tests {
             snapshot_archive_info::SnapshotArchiveInfo,
             snapshot_package::{SnapshotPackage, SnapshotType},
             snapshot_utils::{
-                self, ArchiveFormat, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILE_NAME,
+                self, ArchiveFormat, SnapshotVersion, SNAPSHOT_STATUS_CACHE_FILENAME,
             },
         },
         solana_sdk::hash::Hash,
@@ -335,7 +335,7 @@ mod tests {
         // the source dir for snapshots
         let dummy_slot_deltas: Vec<BankSlotDelta> = vec![];
         snapshot_utils::serialize_snapshot_data_file(
-            &snapshots_dir.join(SNAPSHOT_STATUS_CACHE_FILE_NAME),
+            &snapshots_dir.join(SNAPSHOT_STATUS_CACHE_FILENAME),
             |stream| {
                 serialize_into(stream, &dummy_slot_deltas)?;
                 Ok(())

core/src/system_monitor_service.rs

@@ -117,12 +117,16 @@ pub fn verify_udp_stats_access() -> Result<(), String> {
 }
 
 impl SystemMonitorService {
-    pub fn new(exit: Arc<AtomicBool>, report_os_network_stats: bool) -> Self {
+    pub fn new(
+        exit: Arc<AtomicBool>,
+        report_os_memory_stats: bool,
+        report_os_network_stats: bool,
+    ) -> Self {
         info!("Starting SystemMonitorService");
         let thread_hdl = Builder::new()
             .name("system-monitor".to_string())
             .spawn(move || {
-                Self::run(exit, report_os_network_stats);
+                Self::run(exit, report_os_memory_stats, report_os_network_stats);
             })
             .unwrap();
 
@@ -331,7 +335,7 @@ impl SystemMonitorService {
         }
     }
 
-    pub fn run(exit: Arc<AtomicBool>, report_os_network_stats: bool) {
+    pub fn run(exit: Arc<AtomicBool>, report_os_memory_stats: bool, report_os_network_stats: bool) {
         let mut udp_stats = None;
         let network_limits_timer = AtomicInterval::default();
         let udp_timer = AtomicInterval::default();
@@ -349,7 +353,7 @@ impl SystemMonitorService {
                     Self::process_udp_stats(&mut udp_stats);
                 }
             }
-            if mem_timer.should_update(SAMPLE_INTERVAL_MEM_MS) {
+            if report_os_memory_stats && mem_timer.should_update(SAMPLE_INTERVAL_MEM_MS) {
                 Self::report_mem_stats();
             }
             sleep(SLEEP_INTERVAL);

core/src/tpu.rs

@@ -10,6 +10,7 @@ use {
             GossipVerifiedVoteHashSender, VerifiedVoteSender, VoteTracker,
         },
         fetch_stage::FetchStage,
+        find_packet_sender_stake_stage::FindPacketSenderStakeStage,
         sigverify::TransactionSigVerifier,
         sigverify_stage::SigVerifyStage,
     },
@@ -36,6 +37,9 @@ use {
 
 pub const DEFAULT_TPU_COALESCE_MS: u64 = 5;
 
+// allow multiple connections for NAT and any open/close overlap
+pub const MAX_QUIC_CONNECTIONS_PER_IP: usize = 8;
+
 pub struct TpuSockets {
     pub transactions: Vec<UdpSocket>,
     pub transaction_forwards: Vec<UdpSocket>,
@@ -52,6 +56,8 @@ pub struct Tpu {
     cluster_info_vote_listener: ClusterInfoVoteListener,
     broadcast_stage: BroadcastStage,
     tpu_quic_t: thread::JoinHandle<()>,
+    find_packet_sender_stake_stage: FindPacketSenderStakeStage,
+    vote_find_packet_sender_stake_stage: FindPacketSenderStakeStage,
 }
 
 impl Tpu {
@@ -100,6 +106,26 @@ impl Tpu {
             poh_recorder,
             tpu_coalesce_ms,
         );
+
+        let (find_packet_sender_stake_sender, find_packet_sender_stake_receiver) = unbounded();
+
+        let find_packet_sender_stake_stage = FindPacketSenderStakeStage::new(
+            packet_receiver,
+            find_packet_sender_stake_sender,
+            bank_forks.clone(),
+            cluster_info.clone(),
+        );
+
+        let (vote_find_packet_sender_stake_sender, vote_find_packet_sender_stake_receiver) =
+            unbounded();
+
+        let vote_find_packet_sender_stake_stage = FindPacketSenderStakeStage::new(
+            vote_packet_receiver,
+            vote_find_packet_sender_stake_sender,
+            bank_forks.clone(),
+            cluster_info.clone(),
+        );
+
         let (verified_sender, verified_receiver) = unbounded();
 
         let tpu_quic_t = solana_streamer::quic::spawn_server(
@@ -108,12 +134,13 @@ impl Tpu {
             cluster_info.my_contact_info().tpu.ip(),
             packet_sender,
             exit.clone(),
+            MAX_QUIC_CONNECTIONS_PER_IP,
         )
         .unwrap();
 
         let sigverify_stage = {
             let verifier = TransactionSigVerifier::default();
-            SigVerifyStage::new(packet_receiver, verified_sender, verifier)
+            SigVerifyStage::new(find_packet_sender_stake_receiver, verified_sender, verifier)
         };
 
         let (verified_tpu_vote_packets_sender, verified_tpu_vote_packets_receiver) = unbounded();
@@ -121,7 +148,7 @@ impl Tpu {
         let vote_sigverify_stage = {
             let verifier = TransactionSigVerifier::new_reject_non_vote();
             SigVerifyStage::new(
-                vote_packet_receiver,
+                vote_find_packet_sender_stake_receiver,
                 verified_tpu_vote_packets_sender,
                 verifier,
             )
@@ -175,6 +202,8 @@ impl Tpu {
             cluster_info_vote_listener,
             broadcast_stage,
             tpu_quic_t,
+            find_packet_sender_stake_stage,
+            vote_find_packet_sender_stake_stage,
         }
     }
 
@@ -185,6 +214,8 @@ impl Tpu {
             self.vote_sigverify_stage.join(),
             self.cluster_info_vote_listener.join(),
             self.banking_stage.join(),
+            self.find_packet_sender_stake_stage.join(),
+            self.vote_find_packet_sender_stake_stage.join(),
         ];
         self.tpu_quic_t.join()?;
         let broadcast_result = self.broadcast_stage.join();

core/src/tvu.rs

@@ -16,6 +16,7 @@ use {
         cost_update_service::CostUpdateService,
         drop_bank_service::DropBankService,
         ledger_cleanup_service::LedgerCleanupService,
+        ledger_metric_report_service::LedgerMetricReportService,
         replay_stage::{ReplayStage, ReplayStageConfig},
         retransmit_stage::RetransmitStage,
         rewards_recorder_service::RewardsRecorderSender,
@@ -26,7 +27,7 @@ use {
         voting_service::VotingService,
     },
     crossbeam_channel::{unbounded, Receiver},
-    solana_accountsdb_plugin_manager::block_metadata_notifier_interface::BlockMetadataNotifierLock,
+    solana_geyser_plugin_manager::block_metadata_notifier_interface::BlockMetadataNotifierLock,
     solana_gossip::cluster_info::ClusterInfo,
     solana_ledger::{
         blockstore::Blockstore, blockstore_processor::TransactionStatusSender,
@@ -70,6 +71,7 @@ pub struct Tvu {
     retransmit_stage: RetransmitStage,
     replay_stage: ReplayStage,
     ledger_cleanup_service: Option<LedgerCleanupService>,
+    ledger_metric_report_service: LedgerMetricReportService,
     accounts_background_service: AccountsBackgroundService,
     accounts_hash_verifier: AccountsHashVerifier,
     cost_update_service: CostUpdateService,
@@ -213,14 +215,6 @@ impl Tvu {
 
         let (ledger_cleanup_slot_sender, ledger_cleanup_slot_receiver) = unbounded();
 
-        let snapshot_interval_slots = {
-            if let Some(config) = bank_forks.read().unwrap().snapshot_config() {
-                config.full_snapshot_archive_interval_slots
-            } else {
-                std::u64::MAX
-            }
-        };
-        info!("snapshot_interval_slots: {}", snapshot_interval_slots);
         let (snapshot_config, pending_snapshot_package) = snapshot_config_and_pending_package
             .map(|(snapshot_config, pending_snapshot_package)| {
                 (Some(snapshot_config), Some(pending_snapshot_package))
@@ -357,6 +351,8 @@ impl Tvu {
             )
         });
 
+        let ledger_metric_report_service = LedgerMetricReportService::new(blockstore, exit);
+
         let accounts_background_service = AccountsBackgroundService::new(
             bank_forks.clone(),
             exit,
@@ -373,6 +369,7 @@ impl Tvu {
             retransmit_stage,
             replay_stage,
             ledger_cleanup_service,
+            ledger_metric_report_service,
             accounts_background_service,
             accounts_hash_verifier,
             cost_update_service,
@@ -389,6 +386,7 @@ impl Tvu {
         if self.ledger_cleanup_service.is_some() {
             self.ledger_cleanup_service.unwrap().join()?;
         }
+        self.ledger_metric_report_service.join()?;
         self.accounts_background_service.join()?;
         self.replay_stage.join()?;
         self.accounts_hash_verifier.join()?;

core/src/unprocessed_packet_batches.rs

@@ -0,0 +1,337 @@
+use {
+    retain_mut::RetainMut,
+    solana_perf::packet::{limited_deserialize, Packet, PacketBatch},
+    solana_sdk::{
+        hash::Hash, message::Message, short_vec::decode_shortu16_len, signature::Signature,
+        transaction::VersionedTransaction,
+    },
+    std::{
+        collections::{HashMap, VecDeque},
+        mem::size_of,
+    },
+};
+
+/// Holds deserialized messages, as well as computed message_hash and other things needed to create
+/// SanitizedTransaction
+#[derive(Debug, Default)]
+pub struct DeserializedPacket {
+    #[allow(dead_code)]
+    versioned_transaction: VersionedTransaction,
+
+    #[allow(dead_code)]
+    message_hash: Hash,
+
+    #[allow(dead_code)]
+    is_simple_vote: bool,
+}
+
+/// Defines the type of entry in `UnprocessedPacketBatches`, it holds original packet_batch
+/// for forwarding, as well as `forwarded` flag;
+/// Each packet in packet_batch are deserialized upon receiving, the result are stored in
+/// `DeserializedPacket` in the same order as packets in `packet_batch`.
+#[derive(Debug, Default)]
+pub struct DeserializedPacketBatch {
+    pub packet_batch: PacketBatch,
+    pub forwarded: bool,
+    // indexes of valid packets in batch, and their corresponding deserialized_packet
+    pub unprocessed_packets: HashMap<usize, DeserializedPacket>,
+}
+
+/// References to a packet in `UnprocessedPacketBatches`, where
+/// - batch_index references to `DeserializedPacketBatch`,
+/// - packet_index references to `packet` within `DeserializedPacketBatch.packet_batch`
+#[derive(Debug, Default)]
+pub struct PacketLocator {
+    #[allow(dead_code)]
+    batch_index: usize,
+    #[allow(dead_code)]
+    packet_index: usize,
+}
+
+/// Currently each banking_stage thread has a `UnprocessedPacketBatches` buffer to store
+/// PacketBatch's received from sigverify. Banking thread continuously scans the buffer
+/// to pick proper packets to add to the block.
+#[derive(Default)]
+pub struct UnprocessedPacketBatches(VecDeque<DeserializedPacketBatch>);
+
+impl std::ops::Deref for UnprocessedPacketBatches {
+    type Target = VecDeque<DeserializedPacketBatch>;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl std::ops::DerefMut for UnprocessedPacketBatches {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+impl RetainMut<DeserializedPacketBatch> for UnprocessedPacketBatches {
+    fn retain_mut<F>(&mut self, f: F)
+    where
+        F: FnMut(&mut DeserializedPacketBatch) -> bool,
+    {
+        RetainMut::retain_mut(&mut self.0, f);
+    }
+}
+
+impl FromIterator<DeserializedPacketBatch> for UnprocessedPacketBatches {
+    fn from_iter<I: IntoIterator<Item = DeserializedPacketBatch>>(iter: I) -> Self {
+        Self(iter.into_iter().collect())
+    }
+}
+
+impl UnprocessedPacketBatches {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn with_capacity(capacity: usize) -> Self {
+        UnprocessedPacketBatches(VecDeque::with_capacity(capacity))
+    }
+
+    /// Returns total number of all packets (including unprocessed and processed) in buffer
+    #[allow(dead_code)]
+    fn get_packets_count(&self) -> usize {
+        self.iter()
+            .map(|deserialized_packet_batch| deserialized_packet_batch.packet_batch.packets.len())
+            .sum()
+    }
+
+    /// Returns total number of unprocessed packets in buffer
+    #[allow(dead_code)]
+    fn get_unprocessed_packets_count(&self) -> usize {
+        self.iter()
+            .map(|deserialized_packet_batch| deserialized_packet_batch.unprocessed_packets.len())
+            .sum()
+    }
+
+    /// Iterates the inner `Vec<DeserializedPacketBatch>`.
+    /// Returns the flattened result of mapping each
+    /// `DeserializedPacketBatch` to a list the batch's inner
+    /// packets' sender's stake and their `PacketLocator`'s within the
+    /// `Vec<DeserializedPacketBatch>`.
+    #[allow(dead_code)]
+    fn get_stakes_and_locators(&self) -> (Vec<u64>, Vec<PacketLocator>) {
+        self.iter()
+            .enumerate()
+            .flat_map(|(batch_index, deserialized_packet_batch)| {
+                let packet_batch = &deserialized_packet_batch.packet_batch;
+                deserialized_packet_batch
+                    .unprocessed_packets
+                    .keys()
+                    .map(move |packet_index| {
+                        let p = &packet_batch.packets[*packet_index];
+                        (
+                            p.meta.sender_stake,
+                            PacketLocator {
+                                batch_index,
+                                packet_index: *packet_index,
+                            },
+                        )
+                    })
+            })
+            .unzip()
+    }
+}
+
+impl DeserializedPacketBatch {
+    pub fn new(packet_batch: PacketBatch, packet_indexes: Vec<usize>, forwarded: bool) -> Self {
+        let unprocessed_packets = Self::deserialize_packets(&packet_batch, &packet_indexes);
+        Self {
+            packet_batch,
+            unprocessed_packets,
+            forwarded,
+        }
+    }
+
+    fn deserialize_packets(
+        packet_batch: &PacketBatch,
+        packet_indexes: &[usize],
+    ) -> HashMap<usize, DeserializedPacket> {
+        packet_indexes
+            .iter()
+            .filter_map(|packet_index| {
+                let deserialized_packet =
+                    Self::deserialize_packet(&packet_batch.packets[*packet_index])?;
+                Some((*packet_index, deserialized_packet))
+            })
+            .collect()
+    }
+
+    fn deserialize_packet(packet: &Packet) -> Option<DeserializedPacket> {
+        let versioned_transaction: VersionedTransaction =
+            match limited_deserialize(&packet.data[0..packet.meta.size]) {
+                Ok(tx) => tx,
+                Err(_) => return None,
+            };
+
+        if let Some(message_bytes) = Self::packet_message(packet) {
+            let message_hash = Message::hash_raw_message(message_bytes);
+            let is_simple_vote = packet.meta.is_simple_vote_tx();
+            Some(DeserializedPacket {
+                versioned_transaction,
+                message_hash,
+                is_simple_vote,
+            })
+        } else {
+            None
+        }
+    }
+
+    /// Read the transaction message from packet data
+    pub fn packet_message(packet: &Packet) -> Option<&[u8]> {
+        let (sig_len, sig_size) = decode_shortu16_len(&packet.data).ok()?;
+        let msg_start = sig_len
+            .checked_mul(size_of::<Signature>())
+            .and_then(|v| v.checked_add(sig_size))?;
+        let msg_end = packet.meta.size;
+        Some(&packet.data[msg_start..msg_end])
+    }
+
+    /// Returns whether the given `PacketBatch` has any more remaining unprocessed
+    /// transactions
+    pub fn update_buffered_packets_with_new_unprocessed(
+        &mut self,
+        _original_unprocessed_indexes: &[usize],
+        new_unprocessed_indexes: &[usize],
+    ) -> bool {
+        let has_more_unprocessed_transactions = !new_unprocessed_indexes.is_empty();
+        if has_more_unprocessed_transactions {
+            self.unprocessed_packets
+                .retain(|index, _| new_unprocessed_indexes.contains(index));
+        } else {
+            self.unprocessed_packets.clear();
+        }
+
+        has_more_unprocessed_transactions
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use {
+        super::*,
+        solana_sdk::{signature::Keypair, system_transaction},
+        std::net::IpAddr,
+    };
+
+    fn packet_with_sender_stake(sender_stake: u64, ip: Option<IpAddr>) -> Packet {
+        let tx = system_transaction::transfer(
+            &Keypair::new(),
+            &solana_sdk::pubkey::new_rand(),
+            1,
+            Hash::new_unique(),
+        );
+        let mut packet = Packet::from_data(None, &tx).unwrap();
+        packet.meta.sender_stake = sender_stake;
+        if let Some(ip) = ip {
+            packet.meta.addr = ip;
+        }
+        packet
+    }
+
+    #[test]
+    fn test_packet_message() {
+        let keypair = Keypair::new();
+        let pubkey = solana_sdk::pubkey::new_rand();
+        let blockhash = Hash::new_unique();
+        let transaction = system_transaction::transfer(&keypair, &pubkey, 1, blockhash);
+        let packet = Packet::from_data(None, &transaction).unwrap();
+        assert_eq!(
+            DeserializedPacketBatch::packet_message(&packet)
+                .unwrap()
+                .to_vec(),
+            transaction.message_data()
+        );
+    }
+
+    #[test]
+    fn test_get_packets_count() {
+        // create a buffer with 3 batches, each has 2 packets but only first one is valid
+        let batch_size = 2usize;
+        let batch_count = 3usize;
+        let unprocessed_packet_batches: UnprocessedPacketBatches = (0..batch_count)
+            .map(|_batch_index| {
+                DeserializedPacketBatch::new(
+                    PacketBatch::new(
+                        (0..batch_size)
+                            .map(|packet_index| packet_with_sender_stake(packet_index as u64, None))
+                            .collect(),
+                    ),
+                    vec![0],
+                    false,
+                )
+            })
+            .collect();
+
+        // Assert total packets count, and unprocessed packets count
+        assert_eq!(
+            batch_size * batch_count,
+            unprocessed_packet_batches.get_packets_count()
+        );
+        assert_eq!(
+            batch_count,
+            unprocessed_packet_batches.get_unprocessed_packets_count()
+        );
+    }
+
+    #[test]
+    fn test_get_stakes_and_locators_from_empty_buffer() {
+        let unprocessed_packet_batches = UnprocessedPacketBatches::default();
+        let (stakes, locators) = unprocessed_packet_batches.get_stakes_and_locators();
+
+        assert!(stakes.is_empty());
+        assert!(locators.is_empty());
+    }
+
+    #[test]
+    fn test_get_stakes_and_locators() {
+        solana_logger::setup();
+
+        // setup senders' address and stake
+        let senders: Vec<(IpAddr, u64)> = vec![
+            (IpAddr::from([127, 0, 0, 1]), 1),
+            (IpAddr::from([127, 0, 0, 2]), 2),
+            (IpAddr::from([127, 0, 0, 3]), 3),
+        ];
+        // create a buffer with 3 batches, each has 2 packet from above sender.
+        // buffer looks like:
+        // [127.0.0.1, 127.0.0.2]
+        // [127.0.0.3, 127.0.0.1]
+        // [127.0.0.2, 127.0.0.3]
+        let batch_size = 2usize;
+        let batch_count = 3usize;
+        let unprocessed_packet_batches: UnprocessedPacketBatches = (0..batch_count)
+            .map(|batch_index| {
+                DeserializedPacketBatch::new(
+                    PacketBatch::new(
+                        (0..batch_size)
+                            .map(|packet_index| {
+                                let n = (batch_index * batch_size + packet_index) % senders.len();
+                                packet_with_sender_stake(senders[n].1, Some(senders[n].0))
+                            })
+                            .collect(),
+                    ),
+                    (0..batch_size).collect(),
+                    false,
+                )
+            })
+            .collect();
+
+        let (stakes, locators) = unprocessed_packet_batches.get_stakes_and_locators();
+
+        // Produced stakes and locators should both have "batch_size * batch_count" entries;
+        assert_eq!(batch_size * batch_count, stakes.len());
+        assert_eq!(batch_size * batch_count, locators.len());
+        // Assert stakes and locators are in good order
+        locators.iter().enumerate().for_each(|(index, locator)| {
+            assert_eq!(
+                stakes[index],
+                senders[(locator.batch_index * batch_size + locator.packet_index) % senders.len()]
+                    .1
+            );
+        });
+    }
+}

core/src/validator.rs

@@ -22,8 +22,8 @@ use {
     },
     crossbeam_channel::{bounded, unbounded, Receiver},
     rand::{thread_rng, Rng},
-    solana_accountsdb_plugin_manager::accountsdb_plugin_service::AccountsDbPluginService,
     solana_entry::poh::compute_hash_time_ns,
+    solana_geyser_plugin_manager::geyser_plugin_service::GeyserPluginService,
     solana_gossip::{
         cluster_info::{
             ClusterInfo, Node, DEFAULT_CONTACT_DEBUG_INTERVAL_MILLIS,
@@ -35,8 +35,10 @@ use {
     },
     solana_ledger::{
         bank_forks_utils,
-        blockstore::{Blockstore, BlockstoreSignals, CompletedSlotsReceiver, PurgeType},
-        blockstore_db::{BlockstoreOptions, BlockstoreRecoveryMode},
+        blockstore::{
+            Blockstore, BlockstoreError, BlockstoreSignals, CompletedSlotsReceiver, PurgeType,
+        },
+        blockstore_db::{BlockstoreOptions, BlockstoreRecoveryMode, LedgerColumnOptions},
         blockstore_processor::{self, TransactionStatusSender},
         leader_schedule::FixedSchedule,
         leader_schedule_cache::LeaderScheduleCache,
@@ -119,7 +121,7 @@ pub struct ValidatorConfig {
     pub account_shrink_paths: Option<Vec<PathBuf>>,
     pub rpc_config: JsonRpcConfig,
     pub accountsdb_repl_service_config: Option<AccountsDbReplServiceConfig>,
-    pub accountsdb_plugin_config_files: Option<Vec<PathBuf>>,
+    pub geyser_plugin_config_files: Option<Vec<PathBuf>>,
     pub rpc_addrs: Option<(SocketAddr, SocketAddr)>, // (JsonRpc, JsonRpcPubSub)
     pub pubsub_config: PubSubConfig,
     pub snapshot_config: Option<SnapshotConfig>,
@@ -150,6 +152,7 @@ pub struct ValidatorConfig {
     pub bpf_jit: bool,
     pub send_transaction_service_config: send_transaction_service::Config,
     pub no_poh_speed_test: bool,
+    pub no_os_memory_stats_reporting: bool,
     pub no_os_network_stats_reporting: bool,
     pub poh_pinned_cpu_core: usize,
     pub poh_hashes_per_batch: u64,
@@ -165,6 +168,7 @@ pub struct ValidatorConfig {
     pub no_wait_for_vote_to_start_leader: bool,
     pub accounts_shrink_ratio: AccountShrinkThreshold,
     pub wait_to_vote_slot: Option<Slot>,
+    pub ledger_column_options: LedgerColumnOptions,
 }
 
 impl Default for ValidatorConfig {
@@ -180,7 +184,7 @@ impl Default for ValidatorConfig {
             account_shrink_paths: None,
             rpc_config: JsonRpcConfig::default(),
             accountsdb_repl_service_config: None,
-            accountsdb_plugin_config_files: None,
+            geyser_plugin_config_files: None,
             rpc_addrs: None,
             pubsub_config: PubSubConfig::default(),
             snapshot_config: None,
@@ -210,6 +214,7 @@ impl Default for ValidatorConfig {
             bpf_jit: false,
             send_transaction_service_config: send_transaction_service::Config::default(),
             no_poh_speed_test: true,
+            no_os_memory_stats_reporting: true,
             no_os_network_stats_reporting: true,
             poh_pinned_cpu_core: poh_service::DEFAULT_PINNED_CPU_CORE,
             poh_hashes_per_batch: poh_service::DEFAULT_HASHES_PER_BATCH,
@@ -225,6 +230,7 @@ impl Default for ValidatorConfig {
             accounts_shrink_ratio: AccountShrinkThreshold::default(),
             accounts_db_config: None,
             wait_to_vote_slot: None,
+            ledger_column_options: LedgerColumnOptions::default(),
         }
     }
 }
@@ -264,6 +270,39 @@ impl Default for ValidatorStartProgress {
     }
 }
 
+struct BlockstoreRootScan {
+    thread: Option<JoinHandle<Result<(), BlockstoreError>>>,
+}
+
+impl BlockstoreRootScan {
+    fn new(config: &ValidatorConfig, blockstore: &Arc<Blockstore>, exit: &Arc<AtomicBool>) -> Self {
+        let thread = if config.rpc_addrs.is_some()
+            && config.rpc_config.enable_rpc_transaction_history
+            && config.rpc_config.rpc_scan_and_fix_roots
+        {
+            let blockstore = blockstore.clone();
+            let exit = exit.clone();
+            Some(
+                Builder::new()
+                    .name("blockstore-root-scan".to_string())
+                    .spawn(move || blockstore.scan_and_fix_roots(&exit))
+                    .unwrap(),
+            )
+        } else {
+            None
+        };
+        Self { thread }
+    }
+
+    fn join(self) {
+        if let Some(blockstore_root_scan) = self.thread {
+            if let Err(err) = blockstore_root_scan.join() {
+                warn!("blockstore_root_scan failed to join {:?}", err);
+            }
+        }
+    }
+}
+
 #[derive(Default)]
 struct TransactionHistoryServices {
     transaction_status_sender: Option<TransactionStatusSender>,
@@ -299,7 +338,7 @@ pub struct Validator {
     pub cluster_info: Arc<ClusterInfo>,
     pub bank_forks: Arc<RwLock<BankForks>>,
     accountsdb_repl_service: Option<AccountsDbReplService>,
-    accountsdb_plugin_service: Option<AccountsDbPluginService>,
+    geyser_plugin_service: Option<GeyserPluginService>,
 }
 
 // in the distant future, get rid of ::new()/exit() and use Result properly...
@@ -338,18 +377,16 @@ impl Validator {
 
         let mut bank_notification_senders = Vec::new();
 
-        let accountsdb_plugin_service =
-            if let Some(accountsdb_plugin_config_files) = &config.accountsdb_plugin_config_files {
+        let geyser_plugin_service =
+            if let Some(geyser_plugin_config_files) = &config.geyser_plugin_config_files {
                 let (confirmed_bank_sender, confirmed_bank_receiver) = unbounded();
                 bank_notification_senders.push(confirmed_bank_sender);
-                let result = AccountsDbPluginService::new(
-                    confirmed_bank_receiver,
-                    accountsdb_plugin_config_files,
-                );
+                let result =
+                    GeyserPluginService::new(confirmed_bank_receiver, geyser_plugin_config_files);
                 match result {
-                    Ok(accountsdb_plugin_service) => Some(accountsdb_plugin_service),
+                    Ok(geyser_plugin_service) => Some(geyser_plugin_service),
                     Err(err) => {
-                        error!("Failed to load the AccountsDb plugin: {:?}", err);
+                        error!("Failed to load the Geyser plugin: {:?}", err);
                         abort();
                     }
                 }
@@ -423,46 +460,37 @@ impl Validator {
 
         let accounts_package_channel = unbounded();
 
-        let accounts_update_notifier =
-            accountsdb_plugin_service
-                .as_ref()
-                .and_then(|accountsdb_plugin_service| {
-                    accountsdb_plugin_service.get_accounts_update_notifier()
-                });
+        let accounts_update_notifier = geyser_plugin_service
+            .as_ref()
+            .and_then(|geyser_plugin_service| geyser_plugin_service.get_accounts_update_notifier());
 
-        let transaction_notifier =
-            accountsdb_plugin_service
-                .as_ref()
-                .and_then(|accountsdb_plugin_service| {
-                    accountsdb_plugin_service.get_transaction_notifier()
-                });
+        let transaction_notifier = geyser_plugin_service
+            .as_ref()
+            .and_then(|geyser_plugin_service| geyser_plugin_service.get_transaction_notifier());
 
-        let block_metadata_notifier =
-            accountsdb_plugin_service
-                .as_ref()
-                .and_then(|accountsdb_plugin_service| {
-                    accountsdb_plugin_service.get_block_metadata_notifier()
-                });
+        let block_metadata_notifier = geyser_plugin_service
+            .as_ref()
+            .and_then(|geyser_plugin_service| geyser_plugin_service.get_block_metadata_notifier());
 
         info!(
-            "AccountsDb plugin: accounts_update_notifier: {} transaction_notifier: {}",
+            "Geyser plugin: accounts_update_notifier: {} transaction_notifier: {}",
             accounts_update_notifier.is_some(),
             transaction_notifier.is_some()
         );
 
         let system_monitor_service = Some(SystemMonitorService::new(
             Arc::clone(&exit),
+            !config.no_os_memory_stats_reporting,
             !config.no_os_network_stats_reporting,
         ));
 
         let (
             genesis_config,
-            bank_forks,
+            mut bank_forks,
             blockstore,
             ledger_signal_receiver,
             completed_slots_receiver,
             leader_schedule_cache,
-            last_full_snapshot_slot,
             starting_snapshot_hashes,
             TransactionHistoryServices {
                 transaction_status_sender,
@@ -473,22 +501,46 @@ impl Validator {
                 cache_block_meta_sender,
                 cache_block_meta_service,
             },
-            tower,
-        ) = new_banks_from_ledger(
-            &id,
-            vote_account,
+            blockstore_process_options,
+            blockstore_root_scan,
+        ) = load_blockstore(
             config,
             ledger_path,
-            config.poh_verify,
             &exit,
-            config.enforce_ulimit_nofile,
             &start_progress,
-            config.no_poh_speed_test,
-            accounts_package_channel.0.clone(),
             accounts_update_notifier,
             transaction_notifier,
         );
 
+        let last_full_snapshot_slot = process_blockstore(
+            &blockstore,
+            &mut bank_forks,
+            &leader_schedule_cache,
+            &blockstore_process_options,
+            transaction_status_sender.as_ref(),
+            cache_block_meta_sender.as_ref(),
+            config.snapshot_config.as_ref(),
+            accounts_package_channel.0.clone(),
+            blockstore_root_scan,
+        );
+        let last_full_snapshot_slot =
+            last_full_snapshot_slot.or_else(|| starting_snapshot_hashes.map(|x| x.full.hash.0));
+
+        maybe_warp_slot(config, ledger_path, &mut bank_forks, &leader_schedule_cache);
+
+        let tower = {
+            let restored_tower = Tower::restore(config.tower_storage.as_ref(), &id);
+            if let Ok(tower) = &restored_tower {
+                reconcile_blockstore_roots_with_tower(tower, &blockstore).unwrap_or_else(|err| {
+                    error!("Failed to reconcile blockstore with tower: {:?}", err);
+                    abort()
+                });
+            }
+
+            post_process_restored_tower(restored_tower, &id, vote_account, config, &bank_forks)
+        };
+        info!("Tower state: {:?}", tower);
+
         *start_progress.write().unwrap() = ValidatorStartProgress::StartingServices;
 
         if !config.no_os_network_stats_reporting {
@@ -499,10 +551,6 @@ impl Validator {
         }
 
         let leader_schedule_cache = Arc::new(leader_schedule_cache);
-        let bank = bank_forks.working_bank();
-        if let Some(ref shrink_paths) = config.account_shrink_paths {
-            bank.set_shrink_paths(shrink_paths.clone());
-        }
         let bank_forks = Arc::new(RwLock::new(bank_forks));
 
         let sample_performance_service =
@@ -516,6 +564,7 @@ impl Validator {
                 None
             };
 
+        let bank = bank_forks.read().unwrap().working_bank();
         info!("Starting validator with working bank slot {}", bank.slot());
         {
             let hard_forks: Vec<_> = bank.hard_forks().read().unwrap().iter().copied().collect();
@@ -809,7 +858,8 @@ impl Validator {
 
         let vote_tracker = Arc::<VoteTracker>::default();
         let mut cost_model = CostModel::default();
-        cost_model.initialize_cost_table(&blockstore.read_program_costs().unwrap());
+        // initialize cost model with built-in instruction costs only
+        cost_model.initialize_cost_table(&[]);
         let cost_model = Arc::new(RwLock::new(cost_model));
 
         let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
@@ -937,7 +987,7 @@ impl Validator {
             cluster_info,
             bank_forks,
             accountsdb_repl_service,
-            accountsdb_plugin_service,
+            geyser_plugin_service,
         }
     }
 
@@ -1059,10 +1109,8 @@ impl Validator {
                 .expect("accountsdb_repl_service");
         }
 
-        if let Some(accountsdb_plugin_service) = self.accountsdb_plugin_service {
-            accountsdb_plugin_service
-                .join()
-                .expect("accountsdb_plugin_service");
+        if let Some(geyser_plugin_service) = self.geyser_plugin_service {
+            geyser_plugin_service.join().expect("geyser_plugin_service");
         }
     }
 }
@@ -1194,18 +1242,12 @@ fn post_process_restored_tower(
         })
 }
 
-#[allow(clippy::type_complexity, clippy::too_many_arguments)]
-fn new_banks_from_ledger(
-    validator_identity: &Pubkey,
-    vote_account: &Pubkey,
+#[allow(clippy::type_complexity)]
+fn load_blockstore(
     config: &ValidatorConfig,
     ledger_path: &Path,
-    poh_verify: bool,
     exit: &Arc<AtomicBool>,
-    enforce_ulimit_nofile: bool,
     start_progress: &Arc<RwLock<ValidatorStartProgress>>,
-    no_poh_speed_test: bool,
-    accounts_package_sender: AccountsPackageSender,
     accounts_update_notifier: Option<AccountsUpdateNotifier>,
     transaction_notifier: Option<TransactionNotifierLock>,
 ) -> (
@@ -1215,10 +1257,10 @@ fn new_banks_from_ledger(
     Receiver<bool>,
     CompletedSlotsReceiver,
     LeaderScheduleCache,
-    Option<Slot>,
     Option<StartingSnapshotHashes>,
     TransactionHistoryServices,
-    Tower,
+    blockstore_processor::ProcessOptions,
+    BlockstoreRootScan,
 ) {
     info!("loading ledger from {:?}...", ledger_path);
     *start_progress.write().unwrap() = ValidatorStartProgress::LoadingLedger;
@@ -1242,7 +1284,7 @@ fn new_banks_from_ledger(
         }
     }
 
-    if !no_poh_speed_test {
+    if !config.no_poh_speed_test {
         check_poh_speed(&genesis_config, None);
     }
 
@@ -1255,41 +1297,20 @@ fn new_banks_from_ledger(
         ledger_path,
         BlockstoreOptions {
             recovery_mode: config.wal_recovery_mode.clone(),
-            enforce_ulimit_nofile,
+            column_options: config.ledger_column_options.clone(),
+            enforce_ulimit_nofile: config.enforce_ulimit_nofile,
             ..BlockstoreOptions::default()
         },
     )
     .expect("Failed to open ledger database");
     blockstore.set_no_compaction(config.no_rocksdb_compaction);
 
-    let restored_tower = Tower::restore(config.tower_storage.as_ref(), validator_identity);
-    if let Ok(tower) = &restored_tower {
-        reconcile_blockstore_roots_with_tower(tower, &blockstore).unwrap_or_else(|err| {
-            error!("Failed to reconcile blockstore with tower: {:?}", err);
-            abort()
-        });
-    }
-
     let blockstore = Arc::new(blockstore);
-    let blockstore_root_scan = if config.rpc_addrs.is_some()
-        && config.rpc_config.enable_rpc_transaction_history
-        && config.rpc_config.rpc_scan_and_fix_roots
-    {
-        let blockstore = blockstore.clone();
-        let exit = exit.clone();
-        Some(
-            Builder::new()
-                .name("blockstore-root-scan".to_string())
-                .spawn(move || blockstore.scan_and_fix_roots(&exit))
-                .unwrap(),
-        )
-    } else {
-        None
-    };
+    let blockstore_root_scan = BlockstoreRootScan::new(config, &blockstore, exit);
 
     let process_options = blockstore_processor::ProcessOptions {
         bpf_jit: config.bpf_jit,
-        poh_verify,
+        poh_verify: config.poh_verify,
         dev_halt_at_slot: config.dev_halt_at_slot,
         new_hard_forks: config.new_hard_forks.clone(),
         debug_keys: config.debug_keys.clone(),
@@ -1318,32 +1339,80 @@ fn new_banks_from_ledger(
             TransactionHistoryServices::default()
         };
 
-    let (
-        mut bank_forks,
-        mut leader_schedule_cache,
-        last_full_snapshot_slot,
+    let (mut bank_forks, mut leader_schedule_cache, starting_snapshot_hashes) =
+        bank_forks_utils::load_bank_forks(
+            &genesis_config,
+            &blockstore,
+            config.account_paths.clone(),
+            config.account_shrink_paths.clone(),
+            config.snapshot_config.as_ref(),
+            &process_options,
+            transaction_history_services
+                .cache_block_meta_sender
+                .as_ref(),
+            accounts_update_notifier,
+        );
+
+    leader_schedule_cache.set_fixed_leader_schedule(config.fixed_leader_schedule.clone());
+    bank_forks.set_snapshot_config(config.snapshot_config.clone());
+    bank_forks.set_accounts_hash_interval_slots(config.accounts_hash_interval_slots);
+    if let Some(ref shrink_paths) = config.account_shrink_paths {
+        bank_forks
+            .working_bank()
+            .set_shrink_paths(shrink_paths.clone());
+    }
+
+    (
+        genesis_config,
+        bank_forks,
+        blockstore,
+        ledger_signal_receiver,
+        completed_slots_receiver,
+        leader_schedule_cache,
         starting_snapshot_hashes,
-    ) = bank_forks_utils::load(
-        &genesis_config,
-        &blockstore,
-        config.account_paths.clone(),
-        config.account_shrink_paths.clone(),
-        config.snapshot_config.as_ref(),
+        transaction_history_services,
         process_options,
-        transaction_history_services
-            .transaction_status_sender
-            .as_ref(),
-        transaction_history_services
-            .cache_block_meta_sender
-            .as_ref(),
+        blockstore_root_scan,
+    )
+}
+
+fn process_blockstore(
+    blockstore: &Blockstore,
+    bank_forks: &mut BankForks,
+    leader_schedule_cache: &LeaderScheduleCache,
+    process_options: &blockstore_processor::ProcessOptions,
+    transaction_status_sender: Option<&TransactionStatusSender>,
+    cache_block_meta_sender: Option<&CacheBlockMetaSender>,
+    snapshot_config: Option<&SnapshotConfig>,
+    accounts_package_sender: AccountsPackageSender,
+    blockstore_root_scan: BlockstoreRootScan,
+) -> Option<Slot> {
+    let last_full_snapshot_slot = blockstore_processor::process_blockstore_from_root(
+        blockstore,
+        bank_forks,
+        leader_schedule_cache,
+        process_options,
+        transaction_status_sender,
+        cache_block_meta_sender,
+        snapshot_config,
         accounts_package_sender,
-        accounts_update_notifier,
     )
     .unwrap_or_else(|err| {
         error!("Failed to load ledger: {:?}", err);
         abort()
     });
 
+    blockstore_root_scan.join();
+
+    last_full_snapshot_slot
+}
+
+fn maybe_warp_slot(
+    config: &ValidatorConfig,
+    ledger_path: &Path,
+    bank_forks: &mut BankForks,
+    leader_schedule_cache: &LeaderScheduleCache,
+) {
     if let Some(warp_slot) = config.warp_slot {
         let snapshot_config = config.snapshot_config.as_ref().unwrap_or_else(|| {
             error!("warp slot requires a snapshot config");
@@ -1392,40 +1461,6 @@ fn new_banks_from_ledger(
             full_snapshot_archive_info.path().display()
         );
     }
-
-    let tower = post_process_restored_tower(
-        restored_tower,
-        validator_identity,
-        vote_account,
-        config,
-        &bank_forks,
-    );
-
-    info!("Tower state: {:?}", tower);
-
-    leader_schedule_cache.set_fixed_leader_schedule(config.fixed_leader_schedule.clone());
-
-    bank_forks.set_snapshot_config(config.snapshot_config.clone());
-    bank_forks.set_accounts_hash_interval_slots(config.accounts_hash_interval_slots);
-
-    if let Some(blockstore_root_scan) = blockstore_root_scan {
-        if let Err(err) = blockstore_root_scan.join() {
-            warn!("blockstore_root_scan failed to join {:?}", err);
-        }
-    }
-
-    (
-        genesis_config,
-        bank_forks,
-        blockstore,
-        ledger_signal_receiver,
-        completed_slots_receiver,
-        leader_schedule_cache,
-        last_full_snapshot_slot,
-        starting_snapshot_hashes,
-        transaction_history_services,
-        tower,
-    )
 }
 
 fn blockstore_contains_bad_shred_version(

core/tests/fork-selection.rs

@@ -1,13 +1,13 @@
 //! Fork Selection Simulation
 //!
-//! Description of the algorithm can be found in [docs/src/fork-selection.md](docs/src/fork-selection.md).
+//! Description of the algorithm can be found in [docs/src/cluster/managing-forks.md](docs/src/cluster/managing-forks.md).
 //!
 //! A test library function exists for configuring networks.
 //! ```
 //!     /// * num_partitions - 1 to 100 partitions
 //!     /// * fail_rate - 0 to 1.0 rate of packet receive failure
 //!     /// * delay_count - number of forks to observe before voting
-//!     /// * parasite_rate - number of parasite nodes that vote opposite the greedy choice
+//!     /// * parasite_rate - percentage of parasite nodes that vote opposite the greedy choice
 //!     fn test_with_partitions(num_partitions: usize, fail_rate: f64, delay_count: usize, parasite_rate: f64);
 //! ```
 //! Modify the test function
@@ -497,7 +497,7 @@ fn test_no_partitions() {
 /// * num_partitions - 1 to 100 partitions
 /// * fail_rate - 0 to 1.0 rate of packet receive failure
 /// * delay_count - number of forks to observe before voting
-/// * parasite_rate - number of parasite nodes that vote opposite the greedy choice
+/// * parasite_rate - percentage of parasite nodes that vote opposite the greedy choice
 fn test_with_partitions(
     num_partitions: usize,
     fail_rate: f64,

core/tests/ledger_cleanup.rs

@@ -9,7 +9,10 @@ mod tests {
         solana_core::ledger_cleanup_service::LedgerCleanupService,
         solana_ledger::{
             blockstore::{make_many_slot_shreds, Blockstore},
-            blockstore_db::{BlockstoreOptions, BlockstoreRocksFifoOptions, ShredStorageType},
+            blockstore_db::{
+                BlockstoreOptions, BlockstoreRocksFifoOptions, LedgerColumnOptions,
+                ShredStorageType,
+            },
             get_tmp_ledger_path,
         },
         solana_measure::measure::Measure,
@@ -294,7 +297,7 @@ mod tests {
 
         *time_previous = time_now;
         *storage_previous = storage_now;
-        *data_shred_storage_previous = data_shred_storage_now;
+        *data_shred_storage_previous = data_shred_storage_now.try_into().unwrap();
     }
 
     /// Helper function of the benchmark `test_ledger_cleanup_compaction` which
@@ -348,10 +351,14 @@ mod tests {
             &ledger_path,
             if config.fifo_compaction {
                 BlockstoreOptions {
-                    shred_storage_type: ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions {
-                        shred_data_cf_size: config.shred_data_cf_size,
-                        ..BlockstoreRocksFifoOptions::default()
-                    }),
+                    column_options: LedgerColumnOptions {
+                        shred_storage_type: ShredStorageType::RocksFifo(
+                            BlockstoreRocksFifoOptions {
+                                shred_data_cf_size: config.shred_data_cf_size,
+                                ..BlockstoreRocksFifoOptions::default()
+                            },
+                        ),
+                    },
                     ..BlockstoreOptions::default()
                 }
             } else {

core/tests/snapshots.rs

@@ -175,7 +175,7 @@ mod tests {
 
         let check_hash_calculation = false;
         let full_snapshot_archive_path = snapshot_utils::build_full_snapshot_archive_path(
-            snapshot_archives_dir.to_path_buf(),
+            snapshot_archives_dir,
             old_last_bank.slot(),
             &old_last_bank.get_accounts_hash(),
             ArchiveFormat::TarBzip2,
@@ -432,7 +432,7 @@ mod tests {
                 // Only save off the files returned by `get_snapshot_storages`. This is because
                 // some of the storage entries in the accounts directory may be filtered out by
                 // `get_snapshot_storages()` and will not be included in the snapshot. Ultimately,
-                // this means copying naitvely everything in `accounts_dir` to the `saved_accounts_dir`
+                // this means copying natively everything in `accounts_dir` to the `saved_accounts_dir`
                 // will lead to test failure by mismatch when `saved_accounts_dir` is compared to
                 // the unpacked snapshot later in this test's call to `verify_snapshot_archive()`.
                 for file in snapshot_storage_files {
@@ -461,7 +461,7 @@ mod tests {
                 fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap();
 
                 saved_archive_path = Some(snapshot_utils::build_full_snapshot_archive_path(
-                    snapshot_archives_dir.to_path_buf(),
+                    snapshot_archives_dir,
                     slot,
                     &accounts_hash,
                     ArchiveFormat::TarBzip2,
@@ -544,7 +544,7 @@ mod tests {
         snapshot_utils::serialize_snapshot_data_file(
             &saved_snapshots_dir
                 .path()
-                .join(snapshot_utils::SNAPSHOT_STATUS_CACHE_FILE_NAME),
+                .join(snapshot_utils::SNAPSHOT_STATUS_CACHE_FILENAME),
             |stream| {
                 serialize_into(stream, &[] as &[BankSlotDelta])?;
                 Ok(())

docs/convert-ascii-to-svg.sh

@@ -8,11 +8,17 @@ set -e
 cd "$(dirname "$0")"
 output_dir=static/img
 
+svgbob_cli="$(command -v svgbob_cli || true)"
+if [[ -z "$svgbob_cli" ]]; then
+  svgbob_cli="$(command -v svgbob || true)"
+  [[ -n "$svgbob_cli" ]] || ( echo "svgbob_cli binary not found" && exit 1 )
+fi
+
 mkdir -p "$output_dir"
 
 while read -r bob_file; do
   out_file=$(basename "${bob_file%.*}".svg)
-  svgbob "$bob_file" --output "$output_dir/$out_file"
+  "$svgbob_cli" "$bob_file" --output "$output_dir/$out_file"
 done < <(find art/*.bob)
 
 while read -r msc_file; do

docs/sidebars.js

@@ -80,7 +80,7 @@ module.exports = {
       },
       "developing/test-validator",
       "developing/backwards-compatibility",
-      "developing/plugins/accountsdb_plugin"
+      "developing/plugins/geyser-plugins"
     ],
     Integrating: ["integrations/exchange"],
     Validating: [

docs/src/cluster/rpc-endpoints.md

@@ -36,7 +36,7 @@ public RPC endpoints currently available and recommended for each public cluster
 
 ## Mainnet Beta
 
-#### Endpoints
+#### Endpoints*
 
 - `https://api.mainnet-beta.solana.com` - Solana-hosted api node cluster, backed by a load balancer; rate-limited
 - `https://solana-api.projectserum.com` - Project Serum-hosted api node
@@ -48,3 +48,17 @@ public RPC endpoints currently available and recommended for each public cluster
 - Maximum concurrent connections per IP: 40
 - Maximum connection rate per 10 seconds per IP: 40
 - Maximum amount of data per 30 second: 100 MB
+
+*The public RPC endpoints are not intended for production applications. Please
+use dedicated/private RPC servers when you launch your application, drop NFTs,
+etc. The public services are subject to abuse and rate limits may change
+without prior notice. Likewise, high-traffic websites may be blocked without
+prior notice.
+
+## Common HTTP Error Codes
+
+- 403 -- Your IP address or website has been blocked. It is time to run your own RPC server(s) or find a private service.
+- 429 -- Your IP address is exceeding the rate limits. Slow down! Use the
+[Retry-After](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After)
+HTTP response header to determine how long to wait before making another
+request.

docs/src/clusters.md

@@ -97,8 +97,9 @@ $ solana-validator \
     --identity validator-keypair.json \
     --vote-account vote-account-keypair.json \
     --known-validator 5D1fNXzvv5NjV1ysLjirC4WY92RNsVH18vjmcszZd8on \
-    --known-validator 7XSY3MrYnK8vq693Rju17bbPkCN3Z7KvvfvJx4kdrsSY \
+    --known-validator dDzy5SR3AXdYWVqbDEkVFdvSPCtS9ihF5kJkHCtXoFs \
     --known-validator Ft5fbkqNa76vnsjYNwjDZUXoTWpP7VYm3mtsaQckQADN \
+    --known-validator eoKpUABi59aT4rR9HGS3LcMecfut9x7zJyodWWP43YQ \
     --known-validator 9QxCLckBiJc783jnMvXZubK4wH86Eqqvashtrwvcsgkv \
     --only-known-rpc \
     --ledger ledger \
@@ -116,7 +117,9 @@ The identities of the
 [`--known-validator`s](running-validator/validator-start.md#known-validators) are:
 
 - `5D1fNXzvv5NjV1ysLjirC4WY92RNsVH18vjmcszZd8on` - Solana Labs (testnet.solana.com)
+- `dDzy5SR3AXdYWVqbDEkVFdvSPCtS9ihF5kJkHCtXoFs` - MonkeDAO
 - `Ft5fbkqNa76vnsjYNwjDZUXoTWpP7VYm3mtsaQckQADN` - Certus One
+- `eoKpUABi59aT4rR9HGS3LcMecfut9x7zJyodWWP43YQ` - SerGo
 - `9QxCLckBiJc783jnMvXZubK4wH86Eqqvashtrwvcsgkv` - Algo|Stake
 
 ## Mainnet Beta

docs/src/developing/backwards-compatibility.md

@@ -76,7 +76,7 @@ Major releases:
 - [`solana-program`](https://docs.rs/solana-program/) - Rust SDK for writing programs
 - [`solana-client`](https://docs.rs/solana-client/) - Rust client for connecting to RPC API
 - [`solana-cli-config`](https://docs.rs/solana-cli-config/) - Rust client for managing Solana CLI config files
-- [`solana-accountsdb-plugin-interface`](https://docs.rs/solana-accountsdb-plugin-interface/) - Rust interface for developing Solana AccountsDb plugins.
+- [`solana-geyser-plugin-interface`](https://docs.rs/solana-geyser-plugin-interface/) - Rust interface for developing Solana Geyser plugins.
 
 Patch releases:
 

docs/src/developing/clients/javascript-api.md

@@ -6,6 +6,8 @@ title: Web3 JavaScript API
 
 The Solana-Web3.js library aims to provide complete coverage of Solana. The library was built on top of the [Solana JSON RPC API](https://docs.solana.com/developing/clients/jsonrpc-api).
 
+You can find the full documentation for the `@solana/web3.js` library [here](https://solana-labs.github.io/solana-web3.js/).
+
 ## Common Terminology
 
 | Term | Definition |

docs/src/developing/clients/javascript-reference.md

@@ -4,6 +4,10 @@ title: Web3 API Reference
 
 ## Web3 API Reference Guide
 
+The `@solana/web3.js` library is a package that has coverage over the [Solana JSON RPC API](https://docs.solana.com/developing/clients/jsonrpc-api).
+
+You can find the full documentation for the `@solana/web3.js` library [here](https://solana-labs.github.io/solana-web3.js/).
+
 ## General
 
 ### Connection
@@ -677,8 +681,8 @@ console.log(`Stake balance: ${stakeBalance}`)
 
 // We can verify the state of our stake. This may take some time to become active
 let stakeState = await connection.getStakeActivation(stakeAccount.publicKey);
-console.log(`Stake Stake: ${stakeState.state}`);
-// Stake State: inactive
+console.log(`Stake state: ${stakeState.state}`);
+// Stake state: inactive
 
 // To delegate our stake, we get the current vote accounts and choose the first
 let voteAccounts = await connection.getVoteAccounts();

docs/src/developing/clients/jsonrpc-api.md

@@ -389,6 +389,7 @@ Returns identity and transaction information about a confirmed block in the ledg
   - (optional) `transactionDetails: <string>` - level of transaction detail to return, either "full", "signatures", or "none". If parameter not provided, the default detail level is "full".
   - (optional) `rewards: bool` - whether to populate the `rewards` array. If parameter not provided, the default includes rewards.
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment); "processed" is not supported. If parameter not provided, the default is "finalized".
+  - (optional) `maxSupportedTransactionVersion: <number>` - set the max transaction version to return in responses. If the requested block contains a transaction with a higher version, an error will be returned.
 
 #### Results:
 
@@ -413,6 +414,10 @@ The result field will be an object with the following fields:
       - DEPRECATED: `status: <object>` - Transaction status
         - `"Ok": <null>` - Transaction was successful
         - `"Err": <ERR>` - Transaction failed with TransactionError
+      - `loadedAddresses: <object|undefined>` - Transaction addresses loaded from address lookup tables. Undefined if `maxSupportedTransactionVersion` is not set in request params.
+        - `writable: <array[string]>` - Ordered list of base-58 encoded addresses for writable loaded accounts
+        - `readonly: <array[string]>` - Ordered list of base-58 encoded addresses for readonly loaded accounts
+    - `version: <"legacy"|number|undefined>` - Transaction version. Undefined if `maxSupportedTransactionVersion` is not set in request params.
   - `signatures: <array>` - present if "signatures" are requested for transaction details; an array of signatures strings, corresponding to the transaction order in the block
   - `rewards: <array>` - present if rewards are requested; an array of JSON objects containing:
     - `pubkey: <string>` - The public key, as base-58 encoded string, of the account that received the reward
@@ -559,6 +564,10 @@ The JSON structure of a transaction is defined as follows:
     - `programIdIndex: <number>` - Index into the `message.accountKeys` array indicating the program account that executes this instruction.
     - `accounts: <array[number]>` - List of ordered indices into the `message.accountKeys` array indicating which accounts to pass to the program.
     - `data: <string>` - The program input data encoded in a base-58 string.
+  - `addressTableLookups: <array[object]|undefined>` - List of address table lookups used by a transaction to dynamically load addresses from on-chain address lookup tables. Undefined if `maxSupportedTransactionVersion` is not set.
+    - `accountKey: <string>` - base-58 encoded public key for an address lookup table account.
+    - `writableIndexes: <array[number]>` - List of indices used to load addresses of writable accounts from a lookup table.
+    - `readonlyIndexes: <array[number]>` - List of indices used to load addresses of readonly accounts from a lookup table.
 
 #### Inner Instructions Structure
 
@@ -1522,7 +1531,7 @@ Returns the latest blockhash
 
 - `RpcResponse<object>` - RpcResponse JSON object with `value` field set to a JSON object including:
 - `blockhash: <string>` - a Hash as base-58 encoded string
-- `lastValidBlockHeight: u64` - Slot
+- `lastValidBlockHeight: <u64>` - last [block height](../../terminology.md#block-height) at which the blockhash will be valid
 
 #### Example:
 
@@ -2313,7 +2322,7 @@ Returns the slot leaders for a given slot range
 
 #### Results:
 
-- `<array<string>>` - Node identity public keys as base-58 encoded strings
+- `<array[string]>` - Node identity public keys as base-58 encoded strings
 
 #### Example:
 
@@ -2847,6 +2856,7 @@ Returns transaction details for a confirmed transaction
   - (optional) `encoding: <string>` - encoding for each returned Transaction, either "json", "jsonParsed", "base58" (_slow_), "base64". If parameter not provided, the default encoding is "json".
     "jsonParsed" encoding attempts to use program-specific instruction parsers to return more human-readable and explicit data in the `transaction.message.instructions` list. If "jsonParsed" is requested but a parser cannot be found, the instruction falls back to regular JSON encoding (`accounts`, `data`, and `programIdIndex` fields).
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment); "processed" is not supported. If parameter not provided, the default is "finalized".
+  - (optional) `maxSupportedTransactionVersion: <number>` - set the max transaction version to return in responses. If the requested transaction is a higher version, an error will be returned.
 
 #### Results:
 
@@ -2873,6 +2883,10 @@ Returns transaction details for a confirmed transaction
       - `postBalance: <u64>` - account balance in lamports after the reward was applied
       - `rewardType: <string>` - type of reward: currently only "rent", other types may be added in the future
       - `commission: <u8|undefined>` - vote account commission when the reward was credited, only present for voting and staking rewards
+    - `loadedAddresses: <object|undefined>` - Transaction addresses loaded from address lookup tables. Undefined if `maxSupportedTransactionVersion` is not set in request params.
+      - `writable: <array[string]>` - Ordered list of base-58 encoded addresses for writable loaded accounts
+      - `readonly: <array[string]>` - Ordered list of base-58 encoded addresses for readonly loaded accounts
+  - `version: <"legacy"|number|undefined>` - Transaction version. Undefined if `maxSupportedTransactionVersion` is not set in request params.
 
 #### Example:
 
@@ -3045,7 +3059,7 @@ curl http://localhost:8899 -X POST -H "Content-Type: application/json" -d '
 Result:
 
 ```json
-{ "jsonrpc": "2.0", "result": { "solana-core": "1.10.0" }, "id": 1 }
+{ "jsonrpc": "2.0", "result": { "solana-core": "1.10.6" }, "id": 1 }
 ```
 
 ### getVoteAccounts
@@ -5133,7 +5147,7 @@ The result will be an RpcResponse JSON object with `value` set to a JSON object
 - `blockhash: <string>` - a Hash as base-58 encoded string
 - `feeCalculator: <object>` - FeeCalculator object, the fee schedule for this block hash
 - `lastValidSlot: <u64>` - DEPRECATED - this value is inaccurate and should not be relied upon
-- `lastValidBlockHeight: <u64>` - last [block height](../../terminology.md#block-height) at which a blockhash will be valid
+- `lastValidBlockHeight: <u64>` - last [block height](../../terminology.md#block-height) at which the blockhash will be valid
 
 #### Example:
 

docs/src/developing/plugins/geyser-plugins.md

@@ -1,18 +1,18 @@
 ---
-title: Plugins
+title: Geyser Plugins
 ---
 
 ## Overview
 
 Validators under heavy RPC loads, such as when serving getProgramAccounts calls,
 can fall behind the network. To solve this problem, the validator has been
-enhanced to support a plugin mechanism through which the information about
-accounts and slots can be transmitted to external data stores such as relational
-databases, NoSQL databases or Kafka. RPC services then can be developed to
-consume data from these external data stores with the possibility of more
-flexible and targeted optimizations such as caching and indexing. This allows
-the validator to focus on processing transactions without being slowed down by
-busy RPC requests.
+enhanced to support a plugin mechanism, called a "Geyser" plugin, through which
+the information about accounts, slots, blocks, and transactions can be
+transmitted to external data stores such as relational databases, NoSQL
+databases or Kafka. RPC services then can be developed to consume data from
+these external data stores with the possibility of more flexible and targeted
+optimizations such as caching and indexing. This allows the validator to focus
+on processing transactions without being slowed down by busy RPC requests.
 
 This document describes the interfaces of the plugin and the referential plugin
 implementation for the PostgreSQL database.
@@ -22,24 +22,24 @@ implementation for the PostgreSQL database.
 
 ### Important Crates:
 
-- [`solana-accountsdb-plugin-interface`] — This crate defines the plugin
+- [`solana-geyser-plugin-interface`] — This crate defines the plugin
 interfaces.
 
 - [`solana-accountsdb-plugin-postgres`] — The crate for the referential
 plugin implementation for the PostgreSQL database.
 
-[`solana-accountsdb-plugin-interface`]: https://docs.rs/solana-accountsdb-plugin-interface
+[`solana-geyser-plugin-interface`]: https://docs.rs/solana-geyser-plugin-interface
 [`solana-accountsdb-plugin-postgres`]: https://docs.rs/solana-accountsdb-plugin-postgres
 [`solana-sdk`]: https://docs.rs/solana-sdk
 [`solana-transaction-status`]: https://docs.rs/solana-transaction-status
 
 ## The Plugin Interface
 
-The Plugin interface is declared in [`solana-accountsdb-plugin-interface`]. It
-is defined by the trait `AccountsDbPlugin`. The plugin should implement the
+The Plugin interface is declared in [`solana-geyser-plugin-interface`]. It
+is defined by the trait `GeyserPlugin`. The plugin should implement the
 trait and expose a "C" function `_create_plugin` to return the pointer to this
 trait. For example, in the referential implementation, the following code
-instantiates the PostgreSQL plugin `AccountsDbPluginPostgres ` and returns its
+instantiates the PostgreSQL plugin `GeyserPluginPostgres ` and returns its
 pointer.
 
 ```
@@ -47,10 +47,10 @@ pointer.
 #[allow(improper_ctypes_definitions)]
 /// # Safety
 ///
-/// This function returns the AccountsDbPluginPostgres pointer as trait AccountsDbPlugin.
-pub unsafe extern "C" fn _create_plugin() -> *mut dyn AccountsDbPlugin {
-    let plugin = AccountsDbPluginPostgres::new();
-    let plugin: Box<dyn AccountsDbPlugin> = Box::new(plugin);
+/// This function returns the GeyserPluginPostgres pointer as trait GeyserPlugin.
+pub unsafe extern "C" fn _create_plugin() -> *mut dyn GeyserPlugin {
+    let plugin = GeyserPluginPostgres::new();
+    let plugin: Box<dyn GeyserPlugin> = Box::new(plugin);
     Box::into_raw(plugin)
 }
 ```
@@ -62,7 +62,7 @@ file in JSON5 format. The JSON5 file must have a field `libpath` that points
 to the full path name of the shared library implementing the plugin, and may
 have other configuration information, like connection parameters for the external
 database. The plugin configuration file is specified by the validator's CLI
-parameter `--accountsdb-plugin-config` and the file must be readable to the
+parameter `--geyser-plugin-config` and the file must be readable to the
 validator process.
 
 Please see the [config file](#config) for the referential
@@ -164,11 +164,11 @@ please refer to [`solana-sdk`] and [`solana-transaction-status`]
 
 The `slot` points to the slot the transaction is executed at.
 For more details, please refer to the Rust documentation in
-[`solana-accountsdb-plugin-interface`].
+[`solana-geyser-plugin-interface`].
 
 ## Example PostgreSQL Plugin
 
-The [`solana-accountsdb-plugin-postgres`] crate implements a plugin storing
+The [`solana-accountsdb-plugin-postgres`] repository implements a plugin storing
 account data to a PostgreSQL database to illustrate how a plugin can be
 developed.
 
@@ -182,7 +182,7 @@ configuration file looks like the following:
 
 ```
 {
-	"libpath": "/solana/target/release/libsolana_accountsdb_plugin_postgres.so",
+	"libpath": "/solana/target/release/libsolana_geyser_plugin_postgres.so",
 	"host": "postgres-server",
 	"user": "solana",
 	"port": 5433,
@@ -366,7 +366,7 @@ Then run the script:
 psql -U solana -p 5433 -h 10.138.0.9 -w -d solana -f create_schema.sql
 ```
 
-After this, start the validator with the plugin by using the `--accountsdb-plugin-config`
+After this, start the validator with the plugin by using the `--geyser-plugin-config`
 argument mentioned above.
 
 #### Destroy the Schema Objects

docs/src/developing/runtime-facilities/programs.md

@@ -21,7 +21,7 @@ Create new accounts, allocate account data, assign accounts to owning programs,
 transfer lamports from System Program owned accounts and pay transaction fees.
 
 - Program id: `11111111111111111111111111111111`
-- Instructions: [SystemInstruction](https://docs.rs/solana-sdk/VERSION_FOR_DOCS_RS/solana_sdk/system_instruction/enum.SystemInstruction.html)
+- Instructions: [SystemInstruction](https://docs.rs/solana-program/VERSION_FOR_DOCS_RS/solana_program/system_instruction/enum.SystemInstruction.html)
 
 ## Config Program
 

docs/src/implemented-proposals/instruction_introspection.md

@@ -16,8 +16,8 @@ and received the Message's instruction data inside, and also the index of the cu
 Two helper functions to extract this data can be used:
 
 ```
-fn load_current_index(instruction_data: &[u8]) -> u16;
-fn load_instruction_at(instruction_index: usize, instruction_data: &[u8]) -> Result<Instruction>;
+fn load_current_index_checked(instruction_data: &[u8]) -> u16;
+fn load_instruction_at_checked(instruction_index: usize, instruction_sysvar_account_info: &AccountInfo) -> Result<Instruction>;
 ```
 
 The runtime will recognize this special instruction, serialize the Message instruction data