init

* discard time histogram

* closer to the if

* update

.github/workflows/explorer_preview.yml

@@ -10,6 +10,8 @@ jobs:
     if: ${{ github.event.workflow_run.conclusion == 'success' }}
     steps:
       - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
       - uses: amondnet/vercel-action@v20
         with:
           vercel-token: ${{ secrets.VERCEL_TOKEN }} # Required

.gitignore

@@ -4,6 +4,7 @@
 /solana-metrics/
 /solana-metrics.tar.bz2
 /target/
+/test-ledger/
 
 **/*.rs.bk
 .cargo

.travis.yml

@@ -78,6 +78,7 @@ jobs:
     #    - sudo apt-get install libssl-dev libudev-dev
 
     # docs pull request
+   
     - name: "docs"
       if: type IN (push, pull_request) OR tag IS present
       language: node_js

CONTRIBUTING.md

@@ -74,7 +74,7 @@ minutes to execute. Use that time to write a detailed problem description. Once
 the description is written and CI succeeds, click the "Ready to Review" button
 and add reviewers. Adding reviewers before CI succeeds is a fast path to losing
 reviewer engagement. Not only will they be notified and see the PR is not yet
-ready for them, they will also be bombarded them with additional notifications
+ready for them, they will also be bombarded with additional notifications
 each time you push a commit to get past CI or until they "mute" the PR. Once
 muted, you'll need to reach out over some other medium, such as Discord, to
 request they have another look. When you use draft PRs, no notifications are

Cargo.toml

@@ -46,7 +46,10 @@ members = [
     "poh",
     "poh-bench",
     "program-test",
+    "programs/address-lookup-table",
+    "programs/address-lookup-table-tests",
     "programs/bpf_loader",
+    "programs/bpf_loader/gen-syscall-list",
     "programs/compute-budget",
     "programs/config",
     "programs/stake",
@@ -77,13 +80,10 @@ members = [
     "test-validator",
     "rpc-test",
     "client-test",
+    "zk-token-sdk",
+    "programs/zk-token-proof",
 ]
 
 exclude = [
     "programs/bpf",
 ]
-
-# TODO: Remove once the "simd-accel" feature from the reed-solomon-erasure
-# dependency is supported on Apple M1. v2 of the feature resolver is needed to
-# specify arch-specific features.
-resolver = "2"

README.md

@@ -38,12 +38,6 @@ $ sudo apt-get update
 $ sudo apt-get install libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang make
 ```
 
-On Mac M1s, make sure you set up your terminal & homebrew [to use](https://5balloons.info/correct-way-to-install-and-use-homebrew-on-m1-macs/) Rosetta. You can install it with:
-
-```bash
-$ softwareupdate --install-rosetta
-```
-
 ## **2. Download the source code.**
 
 ```bash
@@ -74,7 +68,7 @@ devnet.solana.com. Runs 24/7. Learn more about the [public clusters](https://doc
 
 # Benchmarking
 
-First install the nightly build of rustc. `cargo bench` requires use of the
+First, install the nightly build of rustc. `cargo bench` requires the use of the
 unstable features only available in the nightly build.
 
 ```bash
@@ -121,12 +115,12 @@ the reader to check and validate their accuracy and truthfulness.
 Furthermore, nothing in this project constitutes a solicitation for
 investment.
 
-Any content produced by SF or developer resources that SF provides, are
+Any content produced by SF or developer resources that SF provides are
 for educational and inspirational purposes only. SF does not encourage,
 induce or sanction the deployment, integration or use of any such
 applications (including the code comprising the Solana blockchain
 protocol) in violation of applicable laws or regulations and hereby
-prohibits any such deployment, integration or use. This includes use of
+prohibits any such deployment, integration or use. This includes the use of
 any such applications by the reader (a) in violation of export control
 or sanctions laws of the United States or any other applicable
 jurisdiction, (b) if the reader is located in or ordinarily resident in
@@ -139,7 +133,7 @@ prohibitions.
 The reader should be aware that U.S. export control and sanctions laws
 prohibit U.S. persons (and other persons that are subject to such laws)
 from transacting with persons in certain countries and territories or
-that are on the SDN list. As a project based primarily on open-source
+that are on the SDN list. As a project-based primarily on open-source
 software, it is possible that such sanctioned persons may nevertheless
 bypass prohibitions, obtain the code comprising the Solana blockchain
 protocol (or other project code or applications) and deploy, integrate,

RELEASE.md

@@ -94,7 +94,7 @@ Alternatively use the Github UI.
     ```
 1. Confirm that your freshly cut release branch is shown as `BETA_CHANNEL` and the previous release branch as `STABLE_CHANNEL`:
     ```
-    ci/channel_info.sh
+    ci/channel-info.sh
     ```
 
 ## Steps to Create a Release
@@ -152,5 +152,5 @@ appearing.  To check for progress:
 [Crates.io](https://crates.io/crates/solana) should have an updated Solana version.  This can take 2-3 hours, and sometimes fails in the `solana-secondary` job.
 If this happens and the error is non-fatal, click "Retry" on the "publish crate" job
 
-### Update software on devnet.solana.com/testnet.solana.com/mainnet-beta.solana.com
-See the documentation at https://github.com/solana-labs/cluster-ops/
+### Update software on testnet.solana.com
+See the documentation at https://github.com/solana-labs/cluster-ops/. devnet.solana.com and mainnet-beta.solana.com run stable releases that have been tested on testnet. Do not update devnet or mainnet-beta with a beta release.

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.9.0"
+version = "1.10.0"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -16,15 +16,15 @@ bs58 = "0.4.0"
 bv = "0.11.1"
 Inflector = "0.11.4"
 lazy_static = "1.4.0"
-serde = "1.0.130"
+serde = "1.0.133"
 serde_derive = "1.0.103"
-serde_json = "1.0.72"
-solana-config-program = { path = "../programs/config", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.9.0" }
+serde_json = "1.0.74"
+solana-config-program = { path = "../programs/config", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-vote-program = { path = "../programs/vote", version = "=1.10.0" }
 spl-token = { version = "=3.2.0", features = ["no-entrypoint"] }
 thiserror = "1.0"
-zstd = "0.9.0"
+zstd = "0.9.2"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-accounts-bench"
-version = "1.9.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,11 +11,11 @@ publish = false
 [dependencies]
 log = "0.4.14"
 rayon = "1.5.1"
-solana-logger = { path = "../logger", version = "=1.9.0" }
-solana-runtime = { path = "../runtime", version = "=1.9.0" }
-solana-measure = { path = "../measure", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-solana-version = { path = "../version", version = "=1.9.0" }
+solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-version = { path = "../version", version = "=1.10.0" }
 clap = "2.33.1"
 
 [package.metadata.docs.rs]

accounts-cluster-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-accounts-cluster-bench"
-version = "1.9.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,24 +13,25 @@ clap = "2.33.1"
 log = "0.4.14"
 rand = "0.7.0"
 rayon = "1.5.1"
-solana-account-decoder = { path = "../account-decoder", version = "=1.9.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.9.0" }
-solana-client = { path = "../client", version = "=1.9.0" }
-solana-core = { path = "../core", version = "=1.9.0" }
-solana-faucet = { path = "../faucet", version = "=1.9.0" }
-solana-gossip = { path = "../gossip", version = "=1.9.0" }
-solana-logger = { path = "../logger", version = "=1.9.0" }
-solana-measure = { path = "../measure", version = "=1.9.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.9.0" }
-solana-runtime = { path = "../runtime", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-solana-streamer = { path = "../streamer", version = "=1.9.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.9.0" }
-solana-version = { path = "../version", version = "=1.9.0" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.10.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.10.0" }
+solana-client = { path = "../client", version = "=1.10.0" }
+solana-faucet = { path = "../faucet", version = "=1.10.0" }
+solana-gossip = { path = "../gossip", version = "=1.10.0" }
+solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-streamer = { path = "../streamer", version = "=1.10.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
+solana-version = { path = "../version", version = "=1.10.0" }
 spl-token = { version = "=3.2.0", features = ["no-entrypoint"] }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "=1.9.0" }
+solana-core = { path = "../core", version = "=1.10.0" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.10.0" }
+solana-test-validator = { path = "../test-validator", version = "=1.10.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accounts-cluster-bench/src/main.rs

@@ -23,6 +23,7 @@ use {
     solana_streamer::socket::SocketAddrSpace,
     solana_transaction_status::parse_token::spl_token_instruction,
     std::{
+        cmp::min,
         net::SocketAddr,
         process::exit,
         sync::{
@@ -156,24 +157,30 @@ fn make_create_message(
 fn make_close_message(
     keypair: &Keypair,
     base_keypair: &Keypair,
-    max_closed_seed: Arc<AtomicU64>,
+    max_created: Arc<AtomicU64>,
+    max_closed: Arc<AtomicU64>,
     num_instructions: usize,
     balance: u64,
     spl_token: bool,
 ) -> Message {
     let instructions: Vec<_> = (0..num_instructions)
         .into_iter()
-        .map(|_| {
+        .filter_map(|_| {
             let program_id = if spl_token {
                 inline_spl_token::id()
             } else {
                 system_program::id()
             };
-            let seed = max_closed_seed.fetch_add(1, Ordering::Relaxed).to_string();
+            let max_created_seed = max_created.load(Ordering::Relaxed);
+            let max_closed_seed = max_closed.load(Ordering::Relaxed);
+            if max_closed_seed >= max_created_seed {
+                return None;
+            }
+            let seed = max_closed.fetch_add(1, Ordering::Relaxed).to_string();
             let address =
                 Pubkey::create_with_seed(&base_keypair.pubkey(), &seed, &program_id).unwrap();
             if spl_token {
-                spl_token_instruction(
+                Some(spl_token_instruction(
                     spl_token::instruction::close_account(
                         &spl_token::id(),
                         &spl_token_pubkey(&address),
@@ -182,16 +189,16 @@ fn make_close_message(
                         &[],
                     )
                     .unwrap(),
-                )
+                ))
             } else {
-                system_instruction::transfer_with_seed(
+                Some(system_instruction::transfer_with_seed(
                     &address,
                     &base_keypair.pubkey(),
                     seed,
                     &program_id,
                     &keypair.pubkey(),
                     balance,
-                )
+                ))
             }
         })
         .collect();
@@ -211,6 +218,7 @@ fn run_accounts_bench(
     maybe_lamports: Option<u64>,
     num_instructions: usize,
     mint: Option<Pubkey>,
+    reclaim_accounts: bool,
 ) {
     assert!(num_instructions > 0);
     let client =
@@ -350,6 +358,7 @@ fn run_accounts_bench(
                             let message = make_close_message(
                                 payer_keypairs[0],
                                 &base_keypair,
+                                seed_tracker.max_created.clone(),
                                 seed_tracker.max_closed.clone(),
                                 1,
                                 min_balance,
@@ -372,7 +381,7 @@ fn run_accounts_bench(
         }
 
         count += 1;
-        if last_log.elapsed().as_millis() > 3000 {
+        if last_log.elapsed().as_millis() > 3000 || count >= iterations {
             info!(
                 "total_accounts_created: {} total_accounts_closed: {} tx_sent_count: {} loop_count: {} balance(s): {:?}",
                 total_accounts_created, total_accounts_closed, tx_sent_count, count, balances
@@ -387,6 +396,83 @@ fn run_accounts_bench(
         }
     }
     executor.close();
+
+    if reclaim_accounts {
+        let executor = TransactionExecutor::new(entrypoint_addr);
+        loop {
+            let max_closed_seed = seed_tracker.max_closed.load(Ordering::Relaxed);
+            let max_created_seed = seed_tracker.max_created.load(Ordering::Relaxed);
+
+            if latest_blockhash.elapsed().as_millis() > 10_000 {
+                blockhash = client.get_latest_blockhash().expect("blockhash");
+                latest_blockhash = Instant::now();
+            }
+            message.recent_blockhash = blockhash;
+            let fee = client
+                .get_fee_for_message(&message)
+                .expect("get_fee_for_message");
+
+            let sigs_len = executor.num_outstanding();
+            if sigs_len < batch_size && max_closed_seed < max_created_seed {
+                let num_to_close = min(
+                    batch_size - sigs_len,
+                    (max_created_seed - max_closed_seed) as usize,
+                );
+                if num_to_close >= payer_keypairs.len() {
+                    info!("closing {} accounts", num_to_close);
+                    let chunk_size = num_to_close / payer_keypairs.len();
+                    info!("{:?} chunk_size", chunk_size);
+                    if chunk_size > 0 {
+                        for (i, keypair) in payer_keypairs.iter().enumerate() {
+                            let txs: Vec<_> = (0..chunk_size)
+                                .into_par_iter()
+                                .filter_map(|_| {
+                                    let message = make_close_message(
+                                        keypair,
+                                        &base_keypair,
+                                        seed_tracker.max_created.clone(),
+                                        seed_tracker.max_closed.clone(),
+                                        num_instructions,
+                                        min_balance,
+                                        mint.is_some(),
+                                    );
+                                    if message.instructions.is_empty() {
+                                        return None;
+                                    }
+                                    let signers: Vec<&Keypair> = vec![keypair, &base_keypair];
+                                    Some(Transaction::new(&signers, message, blockhash))
+                                })
+                                .collect();
+                            balances[i] = balances[i].saturating_sub(fee * txs.len() as u64);
+                            info!("close txs: {}", txs.len());
+                            let new_ids = executor.push_transactions(txs);
+                            info!("close ids: {}", new_ids.len());
+                            tx_sent_count += new_ids.len();
+                            total_accounts_closed += (num_instructions * new_ids.len()) as u64;
+                        }
+                    }
+                }
+            } else {
+                let _ = executor.drain_cleared();
+            }
+            count += 1;
+            if last_log.elapsed().as_millis() > 3000 || max_closed_seed >= max_created_seed {
+                info!(
+                    "total_accounts_closed: {} tx_sent_count: {} loop_count: {} balance(s): {:?}",
+                    total_accounts_closed, tx_sent_count, count, balances
+                );
+                last_log = Instant::now();
+            }
+
+            if max_closed_seed >= max_created_seed {
+                break;
+            }
+            if executor.num_outstanding() >= batch_size {
+                sleep(Duration::from_millis(500));
+            }
+        }
+        executor.close();
+    }
 }
 
 fn main() {
@@ -462,7 +548,7 @@ fn main() {
                 .long("iterations")
                 .takes_value(true)
                 .value_name("NUM")
-                .help("Number of iterations to make"),
+                .help("Number of iterations to make. 0 = unlimited iterations."),
         )
         .arg(
             Arg::with_name("check_gossip")
@@ -475,6 +561,12 @@ fn main() {
                 .takes_value(true)
                 .help("Mint address to initialize account"),
         )
+        .arg(
+            Arg::with_name("reclaim_accounts")
+                .long("reclaim-accounts")
+                .takes_value(false)
+                .help("Reclaim accounts after session ends; incompatible with --iterations 0"),
+        )
         .get_matches();
 
     let skip_gossip = !matches.is_present("check_gossip");
@@ -556,6 +648,7 @@ fn main() {
         lamports,
         num_instructions,
         mint,
+        matches.is_present("reclaim_accounts"),
     );
 }
 
@@ -564,12 +657,18 @@ pub mod test {
     use {
         super::*,
         solana_core::validator::ValidatorConfig,
+        solana_faucet::faucet::run_local_faucet,
         solana_local_cluster::{
             local_cluster::{ClusterConfig, LocalCluster},
             validator_configs::make_identical_validator_configs,
         },
         solana_measure::measure::Measure,
-        solana_sdk::poh_config::PohConfig,
+        solana_sdk::{native_token::sol_to_lamports, poh_config::PohConfig},
+        solana_test_validator::TestValidator,
+        spl_token::{
+            solana_program::program_pack::Pack,
+            state::{Account, Mint},
+        },
     };
 
     #[test]
@@ -605,6 +704,108 @@ pub mod test {
             maybe_lamports,
             num_instructions,
             None,
+            false,
+        );
+        start.stop();
+        info!("{}", start);
+    }
+
+    #[test]
+    fn test_create_then_reclaim_spl_token_accounts() {
+        solana_logger::setup();
+        let mint_keypair = Keypair::new();
+        let mint_pubkey = mint_keypair.pubkey();
+        let faucet_addr = run_local_faucet(mint_keypair, None);
+        let test_validator = TestValidator::with_custom_fees(
+            mint_pubkey,
+            1,
+            Some(faucet_addr),
+            SocketAddrSpace::Unspecified,
+        );
+        let rpc_client =
+            RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
+
+        // Created funder
+        let funder = Keypair::new();
+        let latest_blockhash = rpc_client.get_latest_blockhash().unwrap();
+        let signature = rpc_client
+            .request_airdrop_with_blockhash(
+                &funder.pubkey(),
+                sol_to_lamports(1.0),
+                &latest_blockhash,
+            )
+            .unwrap();
+        rpc_client
+            .confirm_transaction_with_spinner(
+                &signature,
+                &latest_blockhash,
+                CommitmentConfig::confirmed(),
+            )
+            .unwrap();
+
+        // Create Mint
+        let spl_mint_keypair = Keypair::new();
+        let spl_mint_len = Mint::get_packed_len();
+        let spl_mint_rent = rpc_client
+            .get_minimum_balance_for_rent_exemption(spl_mint_len)
+            .unwrap();
+        let transaction = Transaction::new_signed_with_payer(
+            &[
+                system_instruction::create_account(
+                    &funder.pubkey(),
+                    &spl_mint_keypair.pubkey(),
+                    spl_mint_rent,
+                    spl_mint_len as u64,
+                    &inline_spl_token::id(),
+                ),
+                spl_token_instruction(
+                    spl_token::instruction::initialize_mint(
+                        &spl_token::id(),
+                        &spl_token_pubkey(&spl_mint_keypair.pubkey()),
+                        &spl_token_pubkey(&spl_mint_keypair.pubkey()),
+                        None,
+                        2,
+                    )
+                    .unwrap(),
+                ),
+            ],
+            Some(&funder.pubkey()),
+            &[&funder, &spl_mint_keypair],
+            latest_blockhash,
+        );
+        let _sig = rpc_client
+            .send_and_confirm_transaction(&transaction)
+            .unwrap();
+
+        let account_len = Account::get_packed_len();
+        let minimum_balance = rpc_client
+            .get_minimum_balance_for_rent_exemption(account_len)
+            .unwrap();
+
+        let iterations = 5;
+        let batch_size = 100;
+        let close_nth_batch = 0;
+        let num_instructions = 4;
+        let mut start = Measure::start("total accounts run");
+        let keypair0 = Keypair::new();
+        let keypair1 = Keypair::new();
+        let keypair2 = Keypair::new();
+        run_accounts_bench(
+            test_validator
+                .rpc_url()
+                .replace("http://", "")
+                .parse()
+                .unwrap(),
+            faucet_addr,
+            &[&keypair0, &keypair1, &keypair2],
+            iterations,
+            Some(account_len as u64),
+            batch_size,
+            close_nth_batch,
+            Some(minimum_balance),
+            num_instructions,
+            Some(spl_mint_keypair.pubkey()),
+            true,
         );
         start.stop();
         info!("{}", start);

accountsdb-plugin-interface/Cargo.toml

@@ -3,17 +3,17 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-accountsdb-plugin-interface"
 description = "The Solana AccountsDb plugin interface."
-version = "1.9.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
-documentation = "https://docs.rs/solana-validator"
+documentation = "https://docs.rs/solana-accountsdb-plugin-interface"
 
 [dependencies]
 log = "0.4.11"
 thiserror = "1.0.30"
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.9.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-interface/src/accountsdb_plugin_interface.rs

@@ -3,8 +3,8 @@
 /// In addition, the dynamic library must export a "C" function _create_plugin which
 /// creates the implementation of the plugin.
 use {
-    solana_sdk::{signature::Signature, transaction::SanitizedTransaction},
-    solana_transaction_status::TransactionStatusMeta,
+    solana_sdk::{clock::UnixTimestamp, signature::Signature, transaction::SanitizedTransaction},
+    solana_transaction_status::{Reward, TransactionStatusMeta},
     std::{any::Any, error, io},
     thiserror::Error,
 };
@@ -12,54 +12,117 @@ use {
 impl Eq for ReplicaAccountInfo<'_> {}
 
 #[derive(Clone, PartialEq, Debug)]
+/// Information about an account being updated
 pub struct ReplicaAccountInfo<'a> {
+    /// The Pubkey for the account
     pub pubkey: &'a [u8],
+
+    /// The lamports for the account
     pub lamports: u64,
+
+    /// The Pubkey of the owner program account
     pub owner: &'a [u8],
+
+    /// This account's data contains a loaded program (and is now read-only)
     pub executable: bool,
+
+    /// The epoch at which this account will next owe rent
     pub rent_epoch: u64,
+
+    /// The data held in this account.
     pub data: &'a [u8],
+
+    /// A global monotonically increasing atomic number, which can be used
+    /// to tell the order of the account update. For example, when an
+    /// account is updated in the same slot multiple times, the update
+    /// with higher write_version should supersede the one with lower
+    /// write_version.
     pub write_version: u64,
 }
 
+/// A wrapper to future-proof ReplicaAccountInfo handling.
+/// If there were a change to the structure of ReplicaAccountInfo,
+/// there would be new enum entry for the newer version, forcing
+/// plugin implementations to handle the change.
 pub enum ReplicaAccountInfoVersions<'a> {
     V0_0_1(&'a ReplicaAccountInfo<'a>),
 }
 
+/// Information about a transaction
 #[derive(Clone, Debug)]
 pub struct ReplicaTransactionInfo<'a> {
+    /// The first signature of the transaction, used for identifying the transaction.
     pub signature: &'a Signature,
+
+    /// Indicates if the transaction is a simple vote transaction.
     pub is_vote: bool,
+
+    /// The sanitized transaction.
     pub transaction: &'a SanitizedTransaction,
+
+    /// Metadata of the transaction status.
     pub transaction_status_meta: &'a TransactionStatusMeta,
 }
 
+/// A wrapper to future-proof ReplicaTransactionInfo handling.
+/// If there were a change to the structure of ReplicaTransactionInfo,
+/// there would be new enum entry for the newer version, forcing
+/// plugin implementations to handle the change.
 pub enum ReplicaTransactionInfoVersions<'a> {
     V0_0_1(&'a ReplicaTransactionInfo<'a>),
 }
 
+#[derive(Clone, Debug)]
+pub struct ReplicaBlockInfo<'a> {
+    pub slot: u64,
+    pub blockhash: &'a str,
+    pub rewards: &'a [Reward],
+    pub block_time: Option<UnixTimestamp>,
+    pub block_height: Option<u64>,
+}
+
+pub enum ReplicaBlockInfoVersions<'a> {
+    V0_0_1(&'a ReplicaBlockInfo<'a>),
+}
+
+/// Errors returned by plugin calls
 #[derive(Error, Debug)]
 pub enum AccountsDbPluginError {
+    /// Error opening the configuration file; for example, when the file
+    /// is not found or when the validator process has no permission to read it.
     #[error("Error opening config file. Error detail: ({0}).")]
     ConfigFileOpenError(#[from] io::Error),
 
+    /// Error in reading the content of the config file or the content
+    /// is not in the expected format.
     #[error("Error reading config file. Error message: ({msg})")]
     ConfigFileReadError { msg: String },
 
+    /// Error when updating the account.
     #[error("Error updating account. Error message: ({msg})")]
     AccountsUpdateError { msg: String },
 
+    /// Error when updating the slot status
     #[error("Error updating slot status. Error message: ({msg})")]
     SlotStatusUpdateError { msg: String },
 
+    /// Any custom error defined by the plugin.
     #[error("Plugin-defined custom error. Error message: ({0})")]
     Custom(Box<dyn error::Error + Send + Sync>),
 }
 
+/// The current status of a slot
 #[derive(Debug, Clone)]
 pub enum SlotStatus {
+    /// The highest slot of the heaviest fork processed by the node. Ledger state at this slot is
+    /// not derived from a confirmed or finalized block, but if multiple forks are present, is from
+    /// the fork the validator believes is most likely to finalize.
     Processed,
+
+    /// The highest slot having reached max vote lockout.
     Rooted,
+
+    /// The highest slot that has been voted on by supermajority of the cluster, ie. is confirmed.
     Confirmed,
 }
 
@@ -75,6 +138,9 @@ impl SlotStatus {
 
 pub type Result<T> = std::result::Result<T, AccountsDbPluginError>;
 
+/// Defines an AccountsDb plugin, to stream data from the runtime.
+/// AccountsDb plugins must describe desired behavior for load and unload,
+/// as well as how they will handle streamed data.
 pub trait AccountsDbPlugin: Any + Send + Sync + std::fmt::Debug {
     fn name(&self) -> &'static str;
 
@@ -93,6 +159,9 @@ pub trait AccountsDbPlugin: Any + Send + Sync + std::fmt::Debug {
     fn on_unload(&mut self) {}
 
     /// Called when an account is updated at a slot.
+    /// When `is_startup` is true, it indicates the account is loaded from
+    /// snapshots when the validator starts up. When `is_startup` is false,
+    /// the account is updated during transaction processing.
     #[allow(unused_variables)]
     fn update_account(
         &mut self,
@@ -129,6 +198,12 @@ pub trait AccountsDbPlugin: Any + Send + Sync + std::fmt::Debug {
         Ok(())
     }
 
+    /// Called when block's metadata is updated.
+    #[allow(unused_variables)]
+    fn notify_block_metadata(&mut self, blockinfo: ReplicaBlockInfoVersions) -> Result<()> {
+        Ok(())
+    }
+
     /// Check if the plugin is interested in account data
     /// Default is true -- if the plugin is not interested in
     /// account data, please return false.

accountsdb-plugin-manager/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-accountsdb-plugin-manager"
 description = "The Solana AccountsDb plugin manager."
-version = "1.9.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,17 +14,16 @@ bs58 = "0.4.0"
 crossbeam-channel = "0.5"
 libloading = "0.7.2"
 log = "0.4.11"
-serde = "1.0.130"
+serde = "1.0.133"
 serde_derive = "1.0.103"
-serde_json = "1.0.72"
-solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.9.0" }
-solana-logger = { path = "../logger", version = "=1.9.0" }
-solana-measure = { path = "../measure", version = "=1.9.0" }
-solana-metrics = { path = "../metrics", version = "=1.9.0" }
-solana-rpc = { path = "../rpc", version = "=1.9.0" }
-solana-runtime = { path = "../runtime", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.9.0" }
+serde_json = "1.0.74"
+solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-metrics = { path = "../metrics", version = "=1.10.0" }
+solana-rpc = { path = "../rpc", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
 thiserror = "1.0.30"
 
 [package.metadata.docs.rs]

accountsdb-plugin-manager/src/accountsdb_plugin_service.rs

@@ -2,6 +2,8 @@ use {
     crate::{
         accounts_update_notifier::AccountsUpdateNotifierImpl,
         accountsdb_plugin_manager::AccountsDbPluginManager,
+        block_metadata_notifier::BlockMetadataNotifierImpl,
+        block_metadata_notifier_interface::BlockMetadataNotifierLock,
         slot_status_notifier::SlotStatusNotifierImpl, slot_status_observer::SlotStatusObserver,
         transaction_notifier::TransactionNotifierImpl,
     },
@@ -50,6 +52,7 @@ pub struct AccountsDbPluginService {
     plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
     accounts_update_notifier: Option<AccountsUpdateNotifier>,
     transaction_notifier: Option<TransactionNotifierLock>,
+    block_metadata_notifier: Option<BlockMetadataNotifierLock>,
 }
 
 impl AccountsDbPluginService {
@@ -102,17 +105,24 @@ impl AccountsDbPluginService {
                 None
             };
 
-        let slot_status_observer =
-            if account_data_notifications_enabled || transaction_notifications_enabled {
-                let slot_status_notifier = SlotStatusNotifierImpl::new(plugin_manager.clone());
-                let slot_status_notifier = Arc::new(RwLock::new(slot_status_notifier));
+        let (slot_status_observer, block_metadata_notifier): (
+            Option<SlotStatusObserver>,
+            Option<BlockMetadataNotifierLock>,
+        ) = if account_data_notifications_enabled || transaction_notifications_enabled {
+            let slot_status_notifier = SlotStatusNotifierImpl::new(plugin_manager.clone());
+            let slot_status_notifier = Arc::new(RwLock::new(slot_status_notifier));
+            (
                 Some(SlotStatusObserver::new(
                     confirmed_bank_receiver,
                     slot_status_notifier,
-                ))
-            } else {
-                None
-            };
+                )),
+                Some(Arc::new(RwLock::new(BlockMetadataNotifierImpl::new(
+                    plugin_manager.clone(),
+                )))),
+            )
+        } else {
+            (None, None)
+        };
 
         info!("Started AccountsDbPluginService");
         Ok(AccountsDbPluginService {
@@ -120,6 +130,7 @@ impl AccountsDbPluginService {
             plugin_manager,
             accounts_update_notifier,
             transaction_notifier,
+            block_metadata_notifier,
         })
     }
 
@@ -186,6 +197,10 @@ impl AccountsDbPluginService {
         self.transaction_notifier.clone()
     }
 
+    pub fn get_block_metadata_notifier(&self) -> Option<BlockMetadataNotifierLock> {
+        self.block_metadata_notifier.clone()
+    }
+
     pub fn join(self) -> thread::Result<()> {
         if let Some(mut slot_status_observer) = self.slot_status_observer {
             slot_status_observer.join()?;

accountsdb-plugin-manager/src/block_metadata_notifier.rs

@@ -0,0 +1,105 @@
+use {
+    crate::{
+        accountsdb_plugin_manager::AccountsDbPluginManager,
+        block_metadata_notifier_interface::BlockMetadataNotifier,
+    },
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        ReplicaBlockInfo, ReplicaBlockInfoVersions,
+    },
+    solana_measure::measure::Measure,
+    solana_metrics::*,
+    solana_runtime::bank::RewardInfo,
+    solana_sdk::{clock::UnixTimestamp, pubkey::Pubkey},
+    solana_transaction_status::{Reward, Rewards},
+    std::sync::{Arc, RwLock},
+};
+
+pub(crate) struct BlockMetadataNotifierImpl {
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+}
+
+impl BlockMetadataNotifier for BlockMetadataNotifierImpl {
+    /// Notify the block metadata
+    fn notify_block_metadata(
+        &self,
+        slot: u64,
+        blockhash: &str,
+        rewards: &RwLock<Vec<(Pubkey, RewardInfo)>>,
+        block_time: Option<UnixTimestamp>,
+        block_height: Option<u64>,
+    ) {
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+        let rewards = Self::build_rewards(rewards);
+
+        for plugin in plugin_manager.plugins.iter_mut() {
+            let mut measure = Measure::start("accountsdb-plugin-update-slot");
+            let block_info =
+                Self::build_replica_block_info(slot, blockhash, &rewards, block_time, block_height);
+            let block_info = ReplicaBlockInfoVersions::V0_0_1(&block_info);
+            match plugin.notify_block_metadata(block_info) {
+                Err(err) => {
+                    error!(
+                        "Failed to update block metadata at slot {}, error: {} to plugin {}",
+                        slot,
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully updated block metadata at slot {} to plugin {}",
+                        slot,
+                        plugin.name()
+                    );
+                }
+            }
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-update-block-metadata-us",
+                measure.as_us() as usize,
+                1000,
+                1000
+            );
+        }
+    }
+}
+
+impl BlockMetadataNotifierImpl {
+    fn build_rewards(rewards: &RwLock<Vec<(Pubkey, RewardInfo)>>) -> Rewards {
+        let rewards = rewards.read().unwrap();
+        rewards
+            .iter()
+            .map(|(pubkey, reward)| Reward {
+                pubkey: pubkey.to_string(),
+                lamports: reward.lamports,
+                post_balance: reward.post_balance,
+                reward_type: Some(reward.reward_type),
+                commission: reward.commission,
+            })
+            .collect()
+    }
+
+    fn build_replica_block_info<'a>(
+        slot: u64,
+        blockhash: &'a str,
+        rewards: &'a [Reward],
+        block_time: Option<UnixTimestamp>,
+        block_height: Option<u64>,
+    ) -> ReplicaBlockInfo<'a> {
+        ReplicaBlockInfo {
+            slot,
+            blockhash,
+            rewards,
+            block_time,
+            block_height,
+        }
+    }
+
+    pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self {
+        Self { plugin_manager }
+    }
+}

accountsdb-plugin-manager/src/block_metadata_notifier_interface.rs

@@ -0,0 +1,20 @@
+use {
+    solana_runtime::bank::RewardInfo,
+    solana_sdk::{clock::UnixTimestamp, pubkey::Pubkey},
+    std::sync::{Arc, RwLock},
+};
+
+/// Interface for notifying block metadata changes
+pub trait BlockMetadataNotifier {
+    /// Notify the block metadata
+    fn notify_block_metadata(
+        &self,
+        slot: u64,
+        blockhash: &str,
+        rewards: &RwLock<Vec<(Pubkey, RewardInfo)>>,
+        block_time: Option<UnixTimestamp>,
+        block_height: Option<u64>,
+    );
+}
+
+pub type BlockMetadataNotifierLock = Arc<RwLock<dyn BlockMetadataNotifier + Sync + Send>>;

accountsdb-plugin-manager/src/lib.rs

@@ -1,6 +1,8 @@
 pub mod accounts_update_notifier;
 pub mod accountsdb_plugin_manager;
 pub mod accountsdb_plugin_service;
+pub mod block_metadata_notifier;
+pub mod block_metadata_notifier_interface;
 pub mod slot_status_notifier;
 pub mod slot_status_observer;
 pub mod transaction_notifier;

accountsdb-plugin-manager/src/transaction_notifier.rs

@@ -8,7 +8,6 @@ use {
     solana_measure::measure::Measure,
     solana_metrics::*,
     solana_rpc::transaction_notifier_interface::TransactionNotifier,
-    solana_runtime::bank,
     solana_sdk::{clock::Slot, signature::Signature, transaction::SanitizedTransaction},
     solana_transaction_status::TransactionStatusMeta,
     std::sync::{Arc, RwLock},
@@ -85,7 +84,7 @@ impl TransactionNotifierImpl {
     ) -> ReplicaTransactionInfo<'a> {
         ReplicaTransactionInfo {
             signature,
-            is_vote: bank::is_simple_vote_transaction(transaction),
+            is_vote: transaction.is_simple_vote_transaction(),
             transaction,
             transaction_status_meta,
         }

accountsdb-plugin-postgres/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-accountsdb-plugin-postgres"
 description = "The Solana AccountsDb plugin for PostgreSQL database."
-version = "1.9.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -19,21 +19,21 @@ crossbeam-channel = "0.5"
 log = "0.4.14"
 postgres = { version = "0.19.2", features = ["with-chrono-0_4"] }
 postgres-types = { version = "0.2.2", features = ["derive"] }
-serde = "1.0.130"
+serde = "1.0.133"
 serde_derive = "1.0.103"
-serde_json = "1.0.72"
-solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.9.0" }
-solana-logger = { path = "../logger", version = "=1.9.0" }
-solana-measure = { path = "../measure", version = "=1.9.0" }
-solana-metrics = { path = "../metrics", version = "=1.9.0" }
-solana-runtime = { path = "../runtime", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.9.0" }
+serde_json = "1.0.74"
+solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.10.0" }
+solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-metrics = { path = "../metrics", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
 thiserror = "1.0.30"
 tokio-postgres = "0.7.4"
 
 [dev-dependencies]
-solana-account-decoder = { path = "../account-decoder", version = "=1.9.0" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.10.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-postgres/scripts/create_schema.sql

@@ -47,7 +47,14 @@ Create TYPE "TransactionErrorCode" AS ENUM (
     'WouldExceedMaxAccountCostLimit',
     'WouldExceedMaxBlockCostLimit',
     'UnsupportedVersion',
-    'InvalidWritableAccount'
+    'InvalidWritableAccount',
+    'WouldExceedMaxAccountDataCostLimit',
+    'TooManyAccountLocks',
+    'AddressLookupTableNotFound',
+    'InvalidAddressLookupTableOwner',
+    'InvalidAddressLookupTableData',
+    'InvalidAddressLookupTableIndex',
+    'InvalidRentPayingAccount'
 );
 
 CREATE TYPE "TransactionError" AS (
@@ -113,9 +120,10 @@ CREATE TYPE "TransactionMessage" AS (
     instructions "CompiledInstruction"[]
 );
 
-CREATE TYPE "AddressMapIndexes" AS (
-    writable SMALLINT[],
-    readonly SMALLINT[]
+CREATE TYPE "TransactionMessageAddressTableLookup" AS (
+    account_key BYTEA,
+    writable_indexes SMALLINT[],
+    readonly_indexes SMALLINT[]
 );
 
 CREATE TYPE "TransactionMessageV0" AS (
@@ -123,17 +131,17 @@ CREATE TYPE "TransactionMessageV0" AS (
     account_keys BYTEA[],
     recent_blockhash BYTEA,
     instructions "CompiledInstruction"[],
-    address_map_indexes "AddressMapIndexes"[]
+    address_table_lookups "TransactionMessageAddressTableLookup"[]
 );
 
-CREATE TYPE "MappedAddresses" AS (
+CREATE TYPE "LoadedAddresses" AS (
     writable BYTEA[],
     readonly BYTEA[]
 );
 
-CREATE TYPE "MappedMessage" AS (
+CREATE TYPE "LoadedMessageV0" AS (
     message "TransactionMessageV0",
-    mapped_addresses "MappedAddresses"
+    loaded_addresses "LoadedAddresses"
 );
 
 -- The table storing transactions
@@ -143,7 +151,7 @@ CREATE TABLE transaction (
     is_vote BOOL NOT NULL,
     message_type SMALLINT, -- 0: legacy, 1: v0 message
     legacy_message "TransactionMessage",
-    v0_mapped_message "MappedMessage",
+    v0_loaded_message "LoadedMessageV0",
     signatures BYTEA[],
     message_hash BYTEA,
     meta "TransactionStatusMeta",
@@ -151,6 +159,16 @@ CREATE TABLE transaction (
     CONSTRAINT transaction_pk PRIMARY KEY (slot, signature)
 );
 
+-- The table storing block metadata
+CREATE TABLE block (
+    slot BIGINT PRIMARY KEY,
+    blockhash VARCHAR(44),
+    rewards "Reward"[],
+    block_time BIGINT,
+    block_height BIGINT,
+    updated_on TIMESTAMP NOT NULL
+);
+
 /**
  * The following is for keeping historical data for accounts and is not required for plugin to work.
  */

accountsdb-plugin-postgres/scripts/drop_schema.sql

@@ -8,15 +8,16 @@ DROP TABLE account_audit;
 DROP TABLE account;
 DROP TABLE slot;
 DROP TABLE transaction;
+DROP TABLE block;
 
 DROP TYPE "TransactionError" CASCADE;
 DROP TYPE "TransactionErrorCode" CASCADE;
-DROP TYPE "MappedMessage" CASCADE;
-DROP TYPE "MappedAddresses" CASCADE;
+DROP TYPE "LoadedMessageV0" CASCADE;
+DROP TYPE "LoadedAddresses" CASCADE;
 DROP TYPE "TransactionMessageV0" CASCADE;
-DROP TYPE "AddressMapIndexes" CASCADE;
 DROP TYPE "TransactionMessage" CASCADE;
 DROP TYPE "TransactionMessageHeader" CASCADE;
+DROP TYPE "TransactionMessageAddressTableLookup" CASCADE;
 DROP TYPE "TransactionStatusMeta" CASCADE;
 DROP TYPE "RewardType" CASCADE;
 DROP TYPE "Reward" CASCADE;

accountsdb-plugin-postgres/src/accountsdb_plugin_postgres.rs

@@ -12,7 +12,7 @@ use {
     serde_json,
     solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
         AccountsDbPlugin, AccountsDbPluginError, ReplicaAccountInfoVersions,
-        ReplicaTransactionInfoVersions, Result, SlotStatus,
+        ReplicaBlockInfoVersions, ReplicaTransactionInfoVersions, Result, SlotStatus,
     },
     solana_metrics::*,
     std::{fs::File, io::Read},
@@ -41,6 +41,8 @@ pub struct AccountsDbPluginPostgresConfig {
     pub threads: Option<usize>,
     pub batch_size: Option<usize>,
     pub panic_on_db_errors: Option<bool>,
+    /// Indicates if to store historical data for accounts
+    pub store_account_historical_data: Option<bool>,
 }
 
 #[derive(Error, Debug)]
@@ -74,7 +76,7 @@ impl AccountsDbPlugin for AccountsDbPluginPostgres {
     /// Accounts either satisyfing the accounts condition or owners condition will be selected.
     /// When only owners is specified,
     /// all accounts belonging to the owners will be streamed.
-    /// The accounts field support wildcard to select all accounts:
+    /// The accounts field supports wildcard to select all accounts:
     /// "accounts_selector" : {
     ///     "accounts" : \["*"\],
     /// }
@@ -85,6 +87,8 @@ impl AccountsDbPlugin for AccountsDbPluginPostgres {
     /// Please refer to https://docs.rs/postgres/0.19.2/postgres/config/struct.Config.html for the connection configuration.
     /// When `connection_str` is set, the values in "host", "user" and "port" are ignored. If `connection_str` is not given,
     /// `host` and `user` must be given.
+    /// "store_account_historical_data", optional, set it to 'true', to store historical account data to account_audit
+    /// table.
     /// * "threads" optional, specifies the number of worker threads for the plugin. A thread
     /// maintains a PostgreSQL connection to the server. The default is '10'.
     /// * "batch_size" optional, specifies the batch size of bulk insert when the AccountsDb is created
@@ -334,6 +338,31 @@ impl AccountsDbPlugin for AccountsDbPluginPostgres {
         Ok(())
     }
 
+    fn notify_block_metadata(&mut self, block_info: ReplicaBlockInfoVersions) -> Result<()> {
+        match &mut self.client {
+            None => {
+                return Err(AccountsDbPluginError::Custom(Box::new(
+                    AccountsDbPluginPostgresError::DataStoreConnectionError {
+                        msg: "There is no connection to the PostgreSQL database.".to_string(),
+                    },
+                )));
+            }
+            Some(client) => match block_info {
+                ReplicaBlockInfoVersions::V0_0_1(block_info) => {
+                    let result = client.update_block_metadata(block_info);
+
+                    if let Err(err) = result {
+                        return Err(AccountsDbPluginError::SlotStatusUpdateError{
+                                msg: format!("Failed to persist the update of block metadata to the PostgreSQL database. Error: {:?}", err)
+                            });
+                    }
+                }
+            },
+        }
+
+        Ok(())
+    }
+
     /// Check if the plugin is interested in account data
     /// Default is true -- if the plugin is not interested in
     /// account data, please return false.

accountsdb-plugin-postgres/src/postgres_client.rs

@@ -1,4 +1,6 @@
 #![allow(clippy::integer_arithmetic)]
+
+mod postgres_client_block_metadata;
 mod postgres_client_transaction;
 
 /// A concurrent implementation for writing accounts into the PostgreSQL in parallel.
@@ -10,9 +12,10 @@ use {
     crossbeam_channel::{bounded, Receiver, RecvTimeoutError, Sender},
     log::*,
     postgres::{Client, NoTls, Statement},
+    postgres_client_block_metadata::DbBlockInfo,
     postgres_client_transaction::LogTransactionRequest,
     solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
-        AccountsDbPluginError, ReplicaAccountInfo, SlotStatus,
+        AccountsDbPluginError, ReplicaAccountInfo, ReplicaBlockInfo, SlotStatus,
     },
     solana_measure::measure::Measure,
     solana_metrics::*,
@@ -36,6 +39,7 @@ const DEFAULT_THREADS_COUNT: usize = 100;
 const DEFAULT_ACCOUNTS_INSERT_BATCH_SIZE: usize = 10;
 const ACCOUNT_COLUMN_COUNT: usize = 9;
 const DEFAULT_PANIC_ON_DB_ERROR: bool = false;
+const DEFAULT_STORE_ACCOUNT_HISTORICAL_DATA: bool = false;
 
 struct PostgresSqlClientWrapper {
     client: Client,
@@ -44,6 +48,8 @@ struct PostgresSqlClientWrapper {
     update_slot_with_parent_stmt: Statement,
     update_slot_without_parent_stmt: Statement,
     update_transaction_log_stmt: Statement,
+    update_block_metadata_stmt: Statement,
+    insert_account_audit_stmt: Option<Statement>,
 }
 
 pub struct SimplePostgresClient {
@@ -195,6 +201,11 @@ pub trait PostgresClient {
         &mut self,
         transaction_log_info: LogTransactionRequest,
     ) -> Result<(), AccountsDbPluginError>;
+
+    fn update_block_metadata(
+        &mut self,
+        block_info: UpdateBlockMetadataRequest,
+    ) -> Result<(), AccountsDbPluginError>;
 }
 
 impl SimplePostgresClient {
@@ -315,6 +326,28 @@ impl SimplePostgresClient {
         }
     }
 
+    fn build_account_audit_insert_statement(
+        client: &mut Client,
+        config: &AccountsDbPluginPostgresConfig,
+    ) -> Result<Statement, AccountsDbPluginError> {
+        let stmt = "INSERT INTO account_audit (pubkey, slot, owner, lamports, executable, rent_epoch, data, write_version, updated_on) \
+        VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)";
+
+        let stmt = client.prepare(stmt);
+
+        match stmt {
+            Err(err) => {
+                return Err(AccountsDbPluginError::Custom(Box::new(AccountsDbPluginPostgresError::DataSchemaError {
+                    msg: format!(
+                        "Error in preparing for the account_audit update PostgreSQL database: {} host: {:?} user: {:?} config: {:?}",
+                        err, config.host, config.user, config
+                    ),
+                })));
+            }
+            Ok(stmt) => Ok(stmt),
+        }
+    }
+
     fn build_slot_upsert_statement_with_parent(
         client: &mut Client,
         config: &AccountsDbPluginPostgresConfig,
@@ -361,8 +394,8 @@ impl SimplePostgresClient {
         }
     }
 
-    /// Internal function for updating or inserting a single account
-    fn upsert_account_internal(
+    /// Internal function for inserting an account into account_audit table.
+    fn insert_account_audit(
         account: &DbAccountInfo,
         statement: &Statement,
         client: &mut Client,
@@ -370,7 +403,43 @@ impl SimplePostgresClient {
         let lamports = account.lamports() as i64;
         let rent_epoch = account.rent_epoch() as i64;
         let updated_on = Utc::now().naive_utc();
-        let result = client.query(
+        let result = client.execute(
+            statement,
+            &[
+                &account.pubkey(),
+                &account.slot,
+                &account.owner(),
+                &lamports,
+                &account.executable(),
+                &rent_epoch,
+                &account.data(),
+                &account.write_version(),
+                &updated_on,
+            ],
+        );
+
+        if let Err(err) = result {
+            let msg = format!(
+                "Failed to persist the insert of account_audit to the PostgreSQL database. Error: {:?}",
+                err
+            );
+            error!("{}", msg);
+            return Err(AccountsDbPluginError::AccountsUpdateError { msg });
+        }
+        Ok(())
+    }
+
+    /// Internal function for updating or inserting a single account
+    fn upsert_account_internal(
+        account: &DbAccountInfo,
+        statement: &Statement,
+        client: &mut Client,
+        insert_account_audit_stmt: &Option<Statement>,
+    ) -> Result<(), AccountsDbPluginError> {
+        let lamports = account.lamports() as i64;
+        let rent_epoch = account.rent_epoch() as i64;
+        let updated_on = Utc::now().naive_utc();
+        let result = client.execute(
             statement,
             &[
                 &account.pubkey(),
@@ -392,6 +461,11 @@ impl SimplePostgresClient {
             );
             error!("{}", msg);
             return Err(AccountsDbPluginError::AccountsUpdateError { msg });
+        } else if result.unwrap() == 0 && insert_account_audit_stmt.is_some() {
+            // If no records modified (inserted or updated), it is because the account is updated
+            // at an older slot, insert the record directly into the account_audit table.
+            let statement = insert_account_audit_stmt.as_ref().unwrap();
+            Self::insert_account_audit(account, statement, client)?;
         }
 
         Ok(())
@@ -400,9 +474,10 @@ impl SimplePostgresClient {
     /// Update or insert a single account
     fn upsert_account(&mut self, account: &DbAccountInfo) -> Result<(), AccountsDbPluginError> {
         let client = self.client.get_mut().unwrap();
+        let insert_account_audit_stmt = &client.insert_account_audit_stmt;
         let statement = &client.update_account_stmt;
         let client = &mut client.client;
-        Self::upsert_account_internal(account, statement, client)
+        Self::upsert_account_internal(account, statement, client, insert_account_audit_stmt)
     }
 
     /// Insert accounts in batch to reduce network overhead
@@ -478,11 +553,12 @@ impl SimplePostgresClient {
         }
 
         let client = self.client.get_mut().unwrap();
+        let insert_account_audit_stmt = &client.insert_account_audit_stmt;
         let statement = &client.update_account_stmt;
         let client = &mut client.client;
 
         for account in self.pending_account_updates.drain(..) {
-            Self::upsert_account_internal(&account, statement, client)?;
+            Self::upsert_account_internal(&account, statement, client, insert_account_audit_stmt)?;
         }
 
         Ok(())
@@ -501,10 +577,24 @@ impl SimplePostgresClient {
             Self::build_slot_upsert_statement_without_parent(&mut client, config)?;
         let update_transaction_log_stmt =
             Self::build_transaction_info_upsert_statement(&mut client, config)?;
+        let update_block_metadata_stmt =
+            Self::build_block_metadata_upsert_statement(&mut client, config)?;
 
         let batch_size = config
             .batch_size
             .unwrap_or(DEFAULT_ACCOUNTS_INSERT_BATCH_SIZE);
+
+        let store_account_historical_data = config
+            .store_account_historical_data
+            .unwrap_or(DEFAULT_STORE_ACCOUNT_HISTORICAL_DATA);
+
+        let insert_account_audit_stmt = if store_account_historical_data {
+            let stmt = Self::build_account_audit_insert_statement(&mut client, config)?;
+            Some(stmt)
+        } else {
+            None
+        };
+
         info!("Created SimplePostgresClient.");
         Ok(Self {
             batch_size,
@@ -516,6 +606,8 @@ impl SimplePostgresClient {
                 update_slot_with_parent_stmt,
                 update_slot_without_parent_stmt,
                 update_transaction_log_stmt,
+                update_block_metadata_stmt,
+                insert_account_audit_stmt,
             }),
         })
     }
@@ -591,6 +683,13 @@ impl PostgresClient for SimplePostgresClient {
     ) -> Result<(), AccountsDbPluginError> {
         self.log_transaction_impl(transaction_log_info)
     }
+
+    fn update_block_metadata(
+        &mut self,
+        block_info: UpdateBlockMetadataRequest,
+    ) -> Result<(), AccountsDbPluginError> {
+        self.update_block_metadata_impl(block_info)
+    }
 }
 
 struct UpdateAccountRequest {
@@ -604,11 +703,16 @@ struct UpdateSlotRequest {
     slot_status: SlotStatus,
 }
 
+pub struct UpdateBlockMetadataRequest {
+    pub block_info: DbBlockInfo,
+}
+
 #[warn(clippy::large_enum_variant)]
 enum DbWorkItem {
     UpdateAccount(Box<UpdateAccountRequest>),
     UpdateSlot(Box<UpdateSlotRequest>),
     LogTransaction(Box<LogTransactionRequest>),
+    UpdateBlockMetadata(Box<UpdateBlockMetadataRequest>),
 }
 
 impl PostgresClientWorker {
@@ -670,7 +774,20 @@ impl PostgresClientWorker {
                         }
                     }
                     DbWorkItem::LogTransaction(transaction_log_info) => {
-                        self.client.log_transaction(*transaction_log_info)?;
+                        if let Err(err) = self.client.log_transaction(*transaction_log_info) {
+                            error!("Failed to update transaction: ({})", err);
+                            if panic_on_db_errors {
+                                abort();
+                            }
+                        }
+                    }
+                    DbWorkItem::UpdateBlockMetadata(block_info) => {
+                        if let Err(err) = self.client.update_block_metadata(*block_info) {
+                            error!("Failed to update block metadata: ({})", err);
+                            if panic_on_db_errors {
+                                abort();
+                            }
+                        }
                     }
                 },
                 Err(err) => match err {
@@ -863,6 +980,25 @@ impl ParallelPostgresClient {
         Ok(())
     }
 
+    pub fn update_block_metadata(
+        &mut self,
+        block_info: &ReplicaBlockInfo,
+    ) -> Result<(), AccountsDbPluginError> {
+        if let Err(err) = self.sender.send(DbWorkItem::UpdateBlockMetadata(Box::new(
+            UpdateBlockMetadataRequest {
+                block_info: DbBlockInfo::from(block_info),
+            },
+        ))) {
+            return Err(AccountsDbPluginError::SlotStatusUpdateError {
+                msg: format!(
+                    "Failed to update the block metadata at slot {:?}, error: {:?}",
+                    block_info.slot, err
+                ),
+            });
+        }
+        Ok(())
+    }
+
     pub fn notify_end_of_startup(&mut self) -> Result<(), AccountsDbPluginError> {
         info!("Notifying the end of startup");
         // Ensure all items in the queue has been received by the workers

accountsdb-plugin-postgres/src/postgres_client/postgres_client_block_metadata.rs

@@ -0,0 +1,97 @@
+use {
+    crate::{
+        accountsdb_plugin_postgres::{
+            AccountsDbPluginPostgresConfig, AccountsDbPluginPostgresError,
+        },
+        postgres_client::{
+            postgres_client_transaction::DbReward, SimplePostgresClient, UpdateBlockMetadataRequest,
+        },
+    },
+    chrono::Utc,
+    log::*,
+    postgres::{Client, Statement},
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        AccountsDbPluginError, ReplicaBlockInfo,
+    },
+};
+
+#[derive(Clone, Debug)]
+pub struct DbBlockInfo {
+    pub slot: i64,
+    pub blockhash: String,
+    pub rewards: Vec<DbReward>,
+    pub block_time: Option<i64>,
+    pub block_height: Option<i64>,
+}
+
+impl<'a> From<&ReplicaBlockInfo<'a>> for DbBlockInfo {
+    fn from(block_info: &ReplicaBlockInfo) -> Self {
+        Self {
+            slot: block_info.slot as i64,
+            blockhash: block_info.blockhash.to_string(),
+            rewards: block_info.rewards.iter().map(DbReward::from).collect(),
+            block_time: block_info.block_time,
+            block_height: block_info
+                .block_height
+                .map(|block_height| block_height as i64),
+        }
+    }
+}
+
+impl SimplePostgresClient {
+    pub(crate) fn build_block_metadata_upsert_statement(
+        client: &mut Client,
+        config: &AccountsDbPluginPostgresConfig,
+    ) -> Result<Statement, AccountsDbPluginError> {
+        let stmt =
+            "INSERT INTO block (slot, blockhash, rewards, block_time, block_height, updated_on) \
+        VALUES ($1, $2, $3, $4, $5, $6)";
+
+        let stmt = client.prepare(stmt);
+
+        match stmt {
+            Err(err) => {
+                return Err(AccountsDbPluginError::Custom(Box::new(AccountsDbPluginPostgresError::DataSchemaError {
+                    msg: format!(
+                        "Error in preparing for the block metadata update PostgreSQL database: ({}) host: {:?} user: {:?} config: {:?}",
+                        err, config.host, config.user, config
+                    ),
+                })));
+            }
+            Ok(stmt) => Ok(stmt),
+        }
+    }
+
+    pub(crate) fn update_block_metadata_impl(
+        &mut self,
+        block_info: UpdateBlockMetadataRequest,
+    ) -> Result<(), AccountsDbPluginError> {
+        let client = self.client.get_mut().unwrap();
+        let statement = &client.update_block_metadata_stmt;
+        let client = &mut client.client;
+        let updated_on = Utc::now().naive_utc();
+
+        let block_info = block_info.block_info;
+        let result = client.query(
+            statement,
+            &[
+                &block_info.slot,
+                &block_info.blockhash,
+                &block_info.rewards,
+                &block_info.block_time,
+                &block_info.block_height,
+                &updated_on,
+            ],
+        );
+
+        if let Err(err) = result {
+            let msg = format!(
+                "Failed to persist the update of block metadata to the PostgreSQL database. Error: {:?}",
+                err);
+            error!("{}", msg);
+            return Err(AccountsDbPluginError::AccountsUpdateError { msg });
+        }
+
+        Ok(())
+    }
+}

accountsdb-plugin-postgres/src/postgres_client/postgres_client_transaction.rs

@@ -18,8 +18,8 @@ use {
     solana_sdk::{
         instruction::CompiledInstruction,
         message::{
-            v0::{self, AddressMapIndexes},
-            MappedAddresses, MappedMessage, Message, MessageHeader, SanitizedMessage,
+            v0::{self, LoadedAddresses, MessageAddressTableLookup},
+            Message, MessageHeader, SanitizedMessage,
         },
         transaction::TransactionError,
     },
@@ -105,10 +105,11 @@ pub struct DbTransactionMessage {
 }
 
 #[derive(Clone, Debug, ToSql)]
-#[postgres(name = "AddressMapIndexes")]
-pub struct DbAddressMapIndexes {
-    pub writable: Vec<i16>,
-    pub readonly: Vec<i16>,
+#[postgres(name = "TransactionMessageAddressTableLookup")]
+pub struct DbTransactionMessageAddressTableLookup {
+    pub account_key: Vec<u8>,
+    pub writable_indexes: Vec<i16>,
+    pub readonly_indexes: Vec<i16>,
 }
 
 #[derive(Clone, Debug, ToSql)]
@@ -118,21 +119,21 @@ pub struct DbTransactionMessageV0 {
     pub account_keys: Vec<Vec<u8>>,
     pub recent_blockhash: Vec<u8>,
     pub instructions: Vec<DbCompiledInstruction>,
-    pub address_map_indexes: Vec<DbAddressMapIndexes>,
+    pub address_table_lookups: Vec<DbTransactionMessageAddressTableLookup>,
 }
 
 #[derive(Clone, Debug, ToSql)]
-#[postgres(name = "MappedAddresses")]
-pub struct DbMappedAddresses {
+#[postgres(name = "LoadedAddresses")]
+pub struct DbLoadedAddresses {
     pub writable: Vec<Vec<u8>>,
     pub readonly: Vec<Vec<u8>>,
 }
 
 #[derive(Clone, Debug, ToSql)]
-#[postgres(name = "MappedMessage")]
-pub struct DbMappedMessage {
+#[postgres(name = "LoadedMessageV0")]
+pub struct DbLoadedMessageV0 {
     pub message: DbTransactionMessageV0,
-    pub mapped_addresses: DbMappedAddresses,
+    pub loaded_addresses: DbLoadedAddresses,
 }
 
 pub struct DbTransaction {
@@ -141,7 +142,7 @@ pub struct DbTransaction {
     pub slot: i64,
     pub message_type: i16,
     pub legacy_message: Option<DbTransactionMessage>,
-    pub v0_mapped_message: Option<DbMappedMessage>,
+    pub v0_loaded_message: Option<DbLoadedMessageV0>,
     pub message_hash: Vec<u8>,
     pub meta: DbTransactionStatusMeta,
     pub signatures: Vec<Vec<u8>>,
@@ -151,32 +152,33 @@ pub struct LogTransactionRequest {
     pub transaction_info: DbTransaction,
 }
 
-impl From<&AddressMapIndexes> for DbAddressMapIndexes {
-    fn from(address_map_indexes: &AddressMapIndexes) -> Self {
+impl From<&MessageAddressTableLookup> for DbTransactionMessageAddressTableLookup {
+    fn from(address_table_lookup: &MessageAddressTableLookup) -> Self {
         Self {
-            writable: address_map_indexes
-                .writable
+            account_key: address_table_lookup.account_key.as_ref().to_vec(),
+            writable_indexes: address_table_lookup
+                .writable_indexes
                 .iter()
-                .map(|address_idx| *address_idx as i16)
+                .map(|idx| *idx as i16)
                 .collect(),
-            readonly: address_map_indexes
-                .readonly
+            readonly_indexes: address_table_lookup
+                .readonly_indexes
                 .iter()
-                .map(|address_idx| *address_idx as i16)
+                .map(|idx| *idx as i16)
                 .collect(),
         }
     }
 }
 
-impl From<&MappedAddresses> for DbMappedAddresses {
-    fn from(mapped_addresses: &MappedAddresses) -> Self {
+impl From<&LoadedAddresses> for DbLoadedAddresses {
+    fn from(loaded_addresses: &LoadedAddresses) -> Self {
         Self {
-            writable: mapped_addresses
+            writable: loaded_addresses
                 .writable
                 .iter()
                 .map(|pubkey| pubkey.as_ref().to_vec())
                 .collect(),
-            readonly: mapped_addresses
+            readonly: loaded_addresses
                 .readonly
                 .iter()
                 .map(|pubkey| pubkey.as_ref().to_vec())
@@ -243,20 +245,20 @@ impl From<&v0::Message> for DbTransactionMessageV0 {
                 .iter()
                 .map(DbCompiledInstruction::from)
                 .collect(),
-            address_map_indexes: message
-                .address_map_indexes
+            address_table_lookups: message
+                .address_table_lookups
                 .iter()
-                .map(DbAddressMapIndexes::from)
+                .map(DbTransactionMessageAddressTableLookup::from)
                 .collect(),
         }
     }
 }
 
-impl From<&MappedMessage> for DbMappedMessage {
-    fn from(message: &MappedMessage) -> Self {
+impl From<&v0::LoadedMessage> for DbLoadedMessageV0 {
+    fn from(message: &v0::LoadedMessage) -> Self {
         Self {
             message: DbTransactionMessageV0::from(&message.message),
-            mapped_addresses: DbMappedAddresses::from(&message.mapped_addresses),
+            loaded_addresses: DbLoadedAddresses::from(&message.loaded_addresses),
         }
     }
 }
@@ -328,6 +330,14 @@ pub enum DbTransactionErrorCode {
     WouldExceedMaxBlockCostLimit,
     UnsupportedVersion,
     InvalidWritableAccount,
+    WouldExceedMaxAccountDataCostLimit,
+    TooManyAccountLocks,
+    AddressLookupTableNotFound,
+    InvalidAddressLookupTableOwner,
+    InvalidAddressLookupTableData,
+    InvalidAddressLookupTableIndex,
+    InvalidRentPayingAccount,
+    WouldExceedMaxVoteCostLimit,
 }
 
 impl From<&TransactionError> for DbTransactionErrorCode {
@@ -354,8 +364,22 @@ impl From<&TransactionError> for DbTransactionErrorCode {
                 Self::WouldExceedMaxAccountCostLimit
             }
             TransactionError::WouldExceedMaxBlockCostLimit => Self::WouldExceedMaxBlockCostLimit,
+            TransactionError::WouldExceedMaxVoteCostLimit => Self::WouldExceedMaxVoteCostLimit,
             TransactionError::UnsupportedVersion => Self::UnsupportedVersion,
             TransactionError::InvalidWritableAccount => Self::InvalidWritableAccount,
+            TransactionError::WouldExceedMaxAccountDataCostLimit => {
+                Self::WouldExceedMaxAccountDataCostLimit
+            }
+            TransactionError::TooManyAccountLocks => Self::TooManyAccountLocks,
+            TransactionError::AddressLookupTableNotFound => Self::AddressLookupTableNotFound,
+            TransactionError::InvalidAddressLookupTableOwner => {
+                Self::InvalidAddressLookupTableOwner
+            }
+            TransactionError::InvalidAddressLookupTableData => Self::InvalidAddressLookupTableData,
+            TransactionError::InvalidAddressLookupTableIndex => {
+                Self::InvalidAddressLookupTableIndex
+            }
+            TransactionError::InvalidRentPayingAccount => Self::InvalidRentPayingAccount,
         }
     }
 }
@@ -460,8 +484,8 @@ fn build_db_transaction(slot: u64, transaction_info: &ReplicaTransactionInfo) ->
             }
             _ => None,
         },
-        v0_mapped_message: match transaction_info.transaction.message() {
-            SanitizedMessage::V0(mapped_message) => Some(DbMappedMessage::from(mapped_message)),
+        v0_loaded_message: match transaction_info.transaction.message() {
+            SanitizedMessage::V0(loaded_message) => Some(DbLoadedMessageV0::from(loaded_message)),
             _ => None,
         },
         signatures: transaction_info
@@ -485,8 +509,16 @@ impl SimplePostgresClient {
         config: &AccountsDbPluginPostgresConfig,
     ) -> Result<Statement, AccountsDbPluginError> {
         let stmt = "INSERT INTO transaction AS txn (signature, is_vote, slot, message_type, legacy_message, \
-        v0_mapped_message, signatures, message_hash, meta, updated_on) \
-        VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)";
+        v0_loaded_message, signatures, message_hash, meta, updated_on) \
+        VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \
+        ON CONFLICT (slot, signature) DO UPDATE SET is_vote=excluded.is_vote, \
+        message_type=excluded.message_type, \
+        legacy_message=excluded.legacy_message, \
+        v0_loaded_message=excluded.v0_loaded_message, \
+        signatures=excluded.signatures, \
+        message_hash=excluded.message_hash, \
+        meta=excluded.meta, \
+        updated_on=excluded.updated_on";
 
         let stmt = client.prepare(stmt);
 
@@ -521,7 +553,7 @@ impl SimplePostgresClient {
                 &transaction_info.slot,
                 &transaction_info.message_type,
                 &transaction_info.legacy_message,
-                &transaction_info.v0_mapped_message,
+                &transaction_info.v0_loaded_message,
                 &transaction_info.signatures,
                 &transaction_info.message_hash,
                 &transaction_info.meta,
@@ -670,42 +702,44 @@ pub(crate) mod tests {
         check_inner_instructions_equality(&inner_instructions, &db_inner_instructions);
     }
 
-    fn check_address_map_indexes_equality(
-        address_map_indexes: &AddressMapIndexes,
-        db_address_map_indexes: &DbAddressMapIndexes,
+    fn check_address_table_lookups_equality(
+        address_table_lookups: &MessageAddressTableLookup,
+        db_address_table_lookups: &DbTransactionMessageAddressTableLookup,
     ) {
         assert_eq!(
-            address_map_indexes.writable.len(),
-            db_address_map_indexes.writable.len()
+            address_table_lookups.writable_indexes.len(),
+            db_address_table_lookups.writable_indexes.len()
         );
         assert_eq!(
-            address_map_indexes.readonly.len(),
-            db_address_map_indexes.readonly.len()
+            address_table_lookups.readonly_indexes.len(),
+            db_address_table_lookups.readonly_indexes.len()
         );
 
-        for i in 0..address_map_indexes.writable.len() {
+        for i in 0..address_table_lookups.writable_indexes.len() {
             assert_eq!(
-                address_map_indexes.writable[i],
-                db_address_map_indexes.writable[i] as u8
+                address_table_lookups.writable_indexes[i],
+                db_address_table_lookups.writable_indexes[i] as u8
             )
         }
-        for i in 0..address_map_indexes.readonly.len() {
+        for i in 0..address_table_lookups.readonly_indexes.len() {
             assert_eq!(
-                address_map_indexes.readonly[i],
-                db_address_map_indexes.readonly[i] as u8
+                address_table_lookups.readonly_indexes[i],
+                db_address_table_lookups.readonly_indexes[i] as u8
             )
         }
     }
 
     #[test]
-    fn test_transform_address_map_indexes() {
-        let address_map_indexes = AddressMapIndexes {
-            writable: vec![1, 2, 3],
-            readonly: vec![4, 5, 6],
+    fn test_transform_address_table_lookups() {
+        let address_table_lookups = MessageAddressTableLookup {
+            account_key: Pubkey::new_unique(),
+            writable_indexes: vec![1, 2, 3],
+            readonly_indexes: vec![4, 5, 6],
         };
 
-        let db_address_map_indexes = DbAddressMapIndexes::from(&address_map_indexes);
-        check_address_map_indexes_equality(&address_map_indexes, &db_address_map_indexes);
+        let db_address_table_lookups =
+            DbTransactionMessageAddressTableLookup::from(&address_table_lookups);
+        check_address_table_lookups_equality(&address_table_lookups, &db_address_table_lookups);
     }
 
     fn check_reward_equality(reward: &Reward, db_reward: &DbReward) {
@@ -1002,6 +1036,10 @@ pub(crate) mod tests {
                     commission: Some(11),
                 },
             ]),
+            loaded_addresses: LoadedAddresses {
+                writable: vec![Pubkey::new_unique()],
+                readonly: vec![Pubkey::new_unique()],
+            },
         }
     }
 
@@ -1089,7 +1127,7 @@ pub(crate) mod tests {
         check_transaction_message_equality(&message, &db_message);
     }
 
-    fn check_transaction_messagev0_equality(
+    fn check_transaction_message_v0_equality(
         message: &v0::Message,
         db_message: &DbTransactionMessageV0,
     ) {
@@ -1106,18 +1144,18 @@ pub(crate) mod tests {
             );
         }
         assert_eq!(
-            message.address_map_indexes.len(),
-            db_message.address_map_indexes.len()
+            message.address_table_lookups.len(),
+            db_message.address_table_lookups.len()
         );
-        for i in 0..message.address_map_indexes.len() {
-            check_address_map_indexes_equality(
-                &message.address_map_indexes[i],
-                &db_message.address_map_indexes[i],
+        for i in 0..message.address_table_lookups.len() {
+            check_address_table_lookups_equality(
+                &message.address_table_lookups[i],
+                &db_message.address_table_lookups[i],
             );
         }
     }
 
-    fn build_transaction_messagev0() -> v0::Message {
+    fn build_transaction_message_v0() -> v0::Message {
         v0::Message {
             header: MessageHeader {
                 num_readonly_signed_accounts: 2,
@@ -1144,71 +1182,76 @@ pub(crate) mod tests {
                     data: vec![14, 15, 16],
                 },
             ],
-            address_map_indexes: vec![
-                AddressMapIndexes {
-                    writable: vec![0],
-                    readonly: vec![1, 2],
+            address_table_lookups: vec![
+                MessageAddressTableLookup {
+                    account_key: Pubkey::new_unique(),
+                    writable_indexes: vec![0],
+                    readonly_indexes: vec![1, 2],
                 },
-                AddressMapIndexes {
-                    writable: vec![1],
-                    readonly: vec![0, 2],
+                MessageAddressTableLookup {
+                    account_key: Pubkey::new_unique(),
+                    writable_indexes: vec![1],
+                    readonly_indexes: vec![0, 2],
                 },
             ],
         }
     }
 
     #[test]
-    fn test_transform_transaction_messagev0() {
-        let message = build_transaction_messagev0();
+    fn test_transform_transaction_message_v0() {
+        let message = build_transaction_message_v0();
 
         let db_message = DbTransactionMessageV0::from(&message);
-        check_transaction_messagev0_equality(&message, &db_message);
+        check_transaction_message_v0_equality(&message, &db_message);
     }
 
-    fn check_mapped_addresses(
-        mapped_addresses: &MappedAddresses,
-        db_mapped_addresses: &DbMappedAddresses,
+    fn check_loaded_addresses(
+        loaded_addresses: &LoadedAddresses,
+        db_loaded_addresses: &DbLoadedAddresses,
     ) {
         assert_eq!(
-            mapped_addresses.writable.len(),
-            db_mapped_addresses.writable.len()
+            loaded_addresses.writable.len(),
+            db_loaded_addresses.writable.len()
         );
-        for i in 0..mapped_addresses.writable.len() {
+        for i in 0..loaded_addresses.writable.len() {
             assert_eq!(
-                mapped_addresses.writable[i].as_ref(),
-                db_mapped_addresses.writable[i]
+                loaded_addresses.writable[i].as_ref(),
+                db_loaded_addresses.writable[i]
             );
         }
 
         assert_eq!(
-            mapped_addresses.readonly.len(),
-            db_mapped_addresses.readonly.len()
+            loaded_addresses.readonly.len(),
+            db_loaded_addresses.readonly.len()
         );
-        for i in 0..mapped_addresses.readonly.len() {
+        for i in 0..loaded_addresses.readonly.len() {
             assert_eq!(
-                mapped_addresses.readonly[i].as_ref(),
-                db_mapped_addresses.readonly[i]
+                loaded_addresses.readonly[i].as_ref(),
+                db_loaded_addresses.readonly[i]
             );
         }
     }
 
-    fn check_mapped_message_equality(message: &MappedMessage, db_message: &DbMappedMessage) {
-        check_transaction_messagev0_equality(&message.message, &db_message.message);
-        check_mapped_addresses(&message.mapped_addresses, &db_message.mapped_addresses);
+    fn check_loaded_message_v0_equality(
+        message: &v0::LoadedMessage,
+        db_message: &DbLoadedMessageV0,
+    ) {
+        check_transaction_message_v0_equality(&message.message, &db_message.message);
+        check_loaded_addresses(&message.loaded_addresses, &db_message.loaded_addresses);
     }
 
     #[test]
-    fn test_transform_mapped_message() {
-        let message = MappedMessage {
-            message: build_transaction_messagev0(),
-            mapped_addresses: MappedAddresses {
+    fn test_transform_loaded_message_v0() {
+        let message = v0::LoadedMessage {
+            message: build_transaction_message_v0(),
+            loaded_addresses: LoadedAddresses {
                 writable: vec![Pubkey::new_unique(), Pubkey::new_unique()],
                 readonly: vec![Pubkey::new_unique(), Pubkey::new_unique()],
             },
         };
 
-        let db_message = DbMappedMessage::from(&message);
-        check_mapped_message_equality(&message, &db_message);
+        let db_message = DbLoadedMessageV0::from(&message);
+        check_loaded_message_v0_equality(&message, &db_message);
     }
 
     fn check_transaction(
@@ -1229,9 +1272,9 @@ pub(crate) mod tests {
             }
             SanitizedMessage::V0(message) => {
                 assert_eq!(db_transaction.message_type, 1);
-                check_mapped_message_equality(
+                check_loaded_message_v0_equality(
                     message,
-                    db_transaction.v0_mapped_message.as_ref().unwrap(),
+                    db_transaction.v0_loaded_message.as_ref().unwrap(),
                 );
             }
         }
@@ -1298,7 +1341,7 @@ pub(crate) mod tests {
                 Signature::new(&[2u8; 64]),
                 Signature::new(&[3u8; 64]),
             ],
-            message: VersionedMessage::V0(build_transaction_messagev0()),
+            message: VersionedMessage::V0(build_transaction_message_v0()),
         }
     }
 
@@ -1313,7 +1356,7 @@ pub(crate) mod tests {
 
         let transaction =
             SanitizedTransaction::try_create(transaction, message_hash, Some(true), |_message| {
-                Ok(MappedAddresses {
+                Ok(LoadedAddresses {
                     writable: vec![Pubkey::new_unique(), Pubkey::new_unique()],
                     readonly: vec![Pubkey::new_unique(), Pubkey::new_unique()],
                 })

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-banking-bench"
-version = "1.9.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,17 +14,17 @@ crossbeam-channel = "0.5"
 log = "0.4.14"
 rand = "0.7.0"
 rayon = "1.5.1"
-solana-core = { path = "../core", version = "=1.9.0" }
-solana-gossip = { path = "../gossip", version = "=1.9.0" }
-solana-ledger = { path = "../ledger", version = "=1.9.0" }
-solana-logger = { path = "../logger", version = "=1.9.0" }
-solana-measure = { path = "../measure", version = "=1.9.0" }
-solana-perf = { path = "../perf", version = "=1.9.0" }
-solana-poh = { path = "../poh", version = "=1.9.0" }
-solana-runtime = { path = "../runtime", version = "=1.9.0" }
-solana-streamer = { path = "../streamer", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-solana-version = { path = "../version", version = "=1.9.0" }
+solana-core = { path = "../core", version = "=1.10.0" }
+solana-gossip = { path = "../gossip", version = "=1.10.0" }
+solana-ledger = { path = "../ledger", version = "=1.10.0" }
+solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-perf = { path = "../perf", version = "=1.10.0" }
+solana-poh = { path = "../poh", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-streamer = { path = "../streamer", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-version = { path = "../version", version = "=1.10.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -1,11 +1,11 @@
 #![allow(clippy::integer_arithmetic)]
 use {
     clap::{crate_description, crate_name, value_t, App, Arg},
-    crossbeam_channel::unbounded,
+    crossbeam_channel::{unbounded, Receiver},
     log::*,
     rand::{thread_rng, Rng},
     rayon::prelude::*,
-    solana_core::banking_stage::BankingStage,
+    solana_core::{banking_stage::BankingStage, packet_deduper::PacketDeduper},
     solana_gossip::cluster_info::{ClusterInfo, Node},
     solana_ledger::{
         blockstore::Blockstore,
@@ -13,7 +13,7 @@ use {
         get_tmp_ledger_path,
     },
     solana_measure::measure::Measure,
-    solana_perf::packet::to_packets_chunked,
+    solana_perf::packet::to_packet_batches,
     solana_poh::poh_recorder::{create_test_recorder, PohRecorder, WorkingBankEntry},
     solana_runtime::{
         accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks,
@@ -28,7 +28,7 @@ use {
     },
     solana_streamer::socket::SocketAddrSpace,
     std::{
-        sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex, RwLock},
+        sync::{atomic::Ordering, Arc, Mutex, RwLock},
         thread::sleep,
         time::{Duration, Instant},
     },
@@ -212,7 +212,7 @@ fn main() {
         bank.clear_signatures();
     }
 
-    let mut verified: Vec<_> = to_packets_chunked(&transactions, packets_per_chunk);
+    let mut verified: Vec<_> = to_packet_batches(&transactions, packets_per_chunk);
     let ledger_path = get_tmp_ledger_path!();
     {
         let blockstore = Arc::new(
@@ -226,6 +226,7 @@ fn main() {
             SocketAddrSpace::Unspecified,
         );
         let cluster_info = Arc::new(cluster_info);
+        let packet_deduper = PacketDeduper::default();
         let banking_stage = BankingStage::new(
             &cluster_info,
             &poh_recorder,
@@ -235,6 +236,7 @@ fn main() {
             None,
             replay_vote_sender,
             Arc::new(RwLock::new(CostModel::default())),
+            packet_deduper.clone(),
         );
         poh_recorder.lock().unwrap().set_bank(&bank);
 
@@ -349,6 +351,7 @@ fn main() {
             // in this chunk, but since we rotate between CHUNKS then
             // we should clear them by the time we come around again to re-use that chunk.
             bank.clear_signatures();
+            packet_deduper.reset();
             total_us += duration_as_us(&now.elapsed());
             debug!(
                 "time: {} us checked: {} sent: {}",
@@ -364,7 +367,7 @@ fn main() {
                     let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen::<u8>()).collect();
                     tx.signatures[0] = Signature::new(&sig[0..64]);
                 }
-                verified = to_packets_chunked(&transactions.clone(), packets_per_chunk);
+                verified = to_packet_batches(&transactions.clone(), packets_per_chunk);
             }
 
             start += chunk_len;

banks-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-client"
-version = "1.9.0"
+version = "1.10.0"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,16 +12,17 @@ edition = "2021"
 [dependencies]
 borsh = "0.9.1"
 futures = "0.3"
-solana-banks-interface = { path = "../banks-interface", version = "=1.9.0" }
-solana-program = { path = "../sdk/program", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-tarpc = { version = "0.26.2", features = ["full"] }
+solana-banks-interface = { path = "../banks-interface", version = "=1.10.0" }
+solana-program = { path = "../sdk/program", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+tarpc = { version = "0.27.2", features = ["full"] }
+thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "=1.9.0" }
-solana-banks-server = { path = "../banks-server", version = "=1.9.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-banks-server = { path = "../banks-server", version = "=1.10.0" }
 
 [lib]
 crate-type = ["lib"]

banks-client/src/error.rs

@@ -0,0 +1,73 @@
+use {
+    solana_sdk::{transaction::TransactionError, transport::TransportError},
+    std::io,
+    tarpc::client::RpcError,
+    thiserror::Error,
+};
+
+/// Errors from BanksClient
+#[derive(Error, Debug)]
+pub enum BanksClientError {
+    #[error("client error: {0}")]
+    ClientError(&'static str),
+
+    #[error(transparent)]
+    Io(#[from] io::Error),
+
+    #[error(transparent)]
+    RpcError(#[from] RpcError),
+
+    #[error("transport transaction error: {0}")]
+    TransactionError(#[from] TransactionError),
+
+    #[error("simulation error: {err:?}, logs: {logs:?}, units_consumed: {units_consumed:?}")]
+    SimulationError {
+        err: TransactionError,
+        logs: Vec<String>,
+        units_consumed: u64,
+    },
+}
+
+impl BanksClientError {
+    pub fn unwrap(&self) -> TransactionError {
+        match self {
+            BanksClientError::TransactionError(err)
+            | BanksClientError::SimulationError { err, .. } => err.clone(),
+            _ => panic!("unexpected transport error"),
+        }
+    }
+}
+
+impl From<BanksClientError> for io::Error {
+    fn from(err: BanksClientError) -> Self {
+        match err {
+            BanksClientError::ClientError(err) => Self::new(io::ErrorKind::Other, err.to_string()),
+            BanksClientError::Io(err) => err,
+            BanksClientError::RpcError(err) => Self::new(io::ErrorKind::Other, err.to_string()),
+            BanksClientError::TransactionError(err) => {
+                Self::new(io::ErrorKind::Other, err.to_string())
+            }
+            BanksClientError::SimulationError { err, .. } => {
+                Self::new(io::ErrorKind::Other, err.to_string())
+            }
+        }
+    }
+}
+
+impl From<BanksClientError> for TransportError {
+    fn from(err: BanksClientError) -> Self {
+        match err {
+            BanksClientError::ClientError(err) => {
+                Self::IoError(io::Error::new(io::ErrorKind::Other, err.to_string()))
+            }
+            BanksClientError::Io(err) => {
+                Self::IoError(io::Error::new(io::ErrorKind::Other, err.to_string()))
+            }
+            BanksClientError::RpcError(err) => {
+                Self::IoError(io::Error::new(io::ErrorKind::Other, err.to_string()))
+            }
+            BanksClientError::TransactionError(err) => Self::TransactionError(err),
+            BanksClientError::SimulationError { err, .. } => Self::TransactionError(err),
+        }
+    }
+}

banks-client/src/lib.rs

@@ -7,9 +7,10 @@
 
 pub use solana_banks_interface::{BanksClient as TarpcClient, TransactionStatus};
 use {
+    crate::error::BanksClientError,
     borsh::BorshDeserialize,
-    futures::{future::join_all, Future, FutureExt},
-    solana_banks_interface::{BanksRequest, BanksResponse},
+    futures::{future::join_all, Future, FutureExt, TryFutureExt},
+    solana_banks_interface::{BanksRequest, BanksResponse, BanksTransactionResultWithSimulation},
     solana_program::{
         clock::Slot, fee_calculator::FeeCalculator, hash::Hash, program_pack::Pack, pubkey::Pubkey,
         rent::Rent, sysvar::Sysvar,
@@ -20,9 +21,7 @@ use {
         message::Message,
         signature::Signature,
         transaction::{self, Transaction},
-        transport,
     },
-    std::io::{self, Error, ErrorKind},
     tarpc::{
         client::{self, NewClient, RequestDispatch},
         context::{self, Context},
@@ -33,6 +32,8 @@ use {
     tokio_serde::formats::Bincode,
 };
 
+mod error;
+
 // This exists only for backward compatibility
 pub trait BanksClientExt {}
 
@@ -57,8 +58,10 @@ impl BanksClient {
         &mut self,
         ctx: Context,
         transaction: Transaction,
-    ) -> impl Future<Output = io::Result<()>> + '_ {
-        self.inner.send_transaction_with_context(ctx, transaction)
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
+        self.inner
+            .send_transaction_with_context(ctx, transaction)
+            .map_err(Into::into)
     }
 
     #[deprecated(
@@ -69,35 +72,41 @@ impl BanksClient {
         &mut self,
         ctx: Context,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, u64)>> + '_ {
+    ) -> impl Future<Output = Result<(FeeCalculator, Hash, u64), BanksClientError>> + '_ {
         #[allow(deprecated)]
         self.inner
             .get_fees_with_commitment_and_context(ctx, commitment)
+            .map_err(Into::into)
     }
 
     pub fn get_transaction_status_with_context(
         &mut self,
         ctx: Context,
         signature: Signature,
-    ) -> impl Future<Output = io::Result<Option<TransactionStatus>>> + '_ {
+    ) -> impl Future<Output = Result<Option<TransactionStatus>, BanksClientError>> + '_ {
         self.inner
             .get_transaction_status_with_context(ctx, signature)
+            .map_err(Into::into)
     }
 
     pub fn get_slot_with_context(
         &mut self,
         ctx: Context,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<Slot>> + '_ {
-        self.inner.get_slot_with_context(ctx, commitment)
+    ) -> impl Future<Output = Result<Slot, BanksClientError>> + '_ {
+        self.inner
+            .get_slot_with_context(ctx, commitment)
+            .map_err(Into::into)
     }
 
     pub fn get_block_height_with_context(
         &mut self,
         ctx: Context,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<Slot>> + '_ {
-        self.inner.get_block_height_with_context(ctx, commitment)
+    ) -> impl Future<Output = Result<Slot, BanksClientError>> + '_ {
+        self.inner
+            .get_block_height_with_context(ctx, commitment)
+            .map_err(Into::into)
     }
 
     pub fn process_transaction_with_commitment_and_context(
@@ -105,9 +114,26 @@ impl BanksClient {
         ctx: Context,
         transaction: Transaction,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<Option<transaction::Result<()>>>> + '_ {
+    ) -> impl Future<Output = Result<Option<transaction::Result<()>>, BanksClientError>> + '_ {
         self.inner
             .process_transaction_with_commitment_and_context(ctx, transaction, commitment)
+            .map_err(Into::into)
+    }
+
+    pub fn process_transaction_with_preflight_and_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = Result<BanksTransactionResultWithSimulation, BanksClientError>> + '_
+    {
+        self.inner
+            .process_transaction_with_preflight_and_commitment_and_context(
+                ctx,
+                transaction,
+                commitment,
+            )
+            .map_err(Into::into)
     }
 
     pub fn get_account_with_commitment_and_context(
@@ -115,9 +141,10 @@ impl BanksClient {
         ctx: Context,
         address: Pubkey,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+    ) -> impl Future<Output = Result<Option<Account>, BanksClientError>> + '_ {
         self.inner
             .get_account_with_commitment_and_context(ctx, address, commitment)
+            .map_err(Into::into)
     }
 
     /// Send a transaction and return immediately. The server will resend the
@@ -126,7 +153,7 @@ impl BanksClient {
     pub fn send_transaction(
         &mut self,
         transaction: Transaction,
-    ) -> impl Future<Output = io::Result<()>> + '_ {
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
         self.send_transaction_with_context(context::current(), transaction)
     }
 
@@ -139,23 +166,25 @@ impl BanksClient {
     )]
     pub fn get_fees(
         &mut self,
-    ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, u64)>> + '_ {
+    ) -> impl Future<Output = Result<(FeeCalculator, Hash, u64), BanksClientError>> + '_ {
         #[allow(deprecated)]
         self.get_fees_with_commitment_and_context(context::current(), CommitmentLevel::default())
     }
 
     /// Return the cluster Sysvar
-    pub fn get_sysvar<T: Sysvar>(&mut self) -> impl Future<Output = io::Result<T>> + '_ {
+    pub fn get_sysvar<T: Sysvar>(
+        &mut self,
+    ) -> impl Future<Output = Result<T, BanksClientError>> + '_ {
         self.get_account(T::id()).map(|result| {
-            let sysvar = result?
-                .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Sysvar not present"))?;
-            from_account::<T, _>(&sysvar)
-                .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Failed to deserialize sysvar"))
+            let sysvar = result?.ok_or(BanksClientError::ClientError("Sysvar not present"))?;
+            from_account::<T, _>(&sysvar).ok_or(BanksClientError::ClientError(
+                "Failed to deserialize sysvar",
+            ))
         })
     }
 
     /// Return the cluster rent
-    pub fn get_rent(&mut self) -> impl Future<Output = io::Result<Rent>> + '_ {
+    pub fn get_rent(&mut self) -> impl Future<Output = Result<Rent, BanksClientError>> + '_ {
         self.get_sysvar::<Rent>()
     }
 
@@ -163,8 +192,11 @@ impl BanksClient {
     /// transactions with a blockhash that has not yet expired. Use the `get_fees`
     /// method to get both a blockhash and the blockhash's last valid slot.
     #[deprecated(since = "1.9.0", note = "Please use `get_latest_blockhash` instead")]
-    pub fn get_recent_blockhash(&mut self) -> impl Future<Output = io::Result<Hash>> + '_ {
-        self.get_latest_blockhash()
+    pub fn get_recent_blockhash(
+        &mut self,
+    ) -> impl Future<Output = Result<Hash, BanksClientError>> + '_ {
+        #[allow(deprecated)]
+        self.get_fees().map(|result| Ok(result?.1))
     }
 
     /// Send a transaction and return after the transaction has been rejected or
@@ -173,23 +205,71 @@ impl BanksClient {
         &mut self,
         transaction: Transaction,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = transport::Result<()>> + '_ {
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
         let mut ctx = context::current();
         ctx.deadline += Duration::from_secs(50);
         self.process_transaction_with_commitment_and_context(ctx, transaction, commitment)
             .map(|result| match result? {
-                None => {
-                    Err(Error::new(ErrorKind::TimedOut, "invalid blockhash or fee-payer").into())
-                }
+                None => Err(BanksClientError::ClientError(
+                    "invalid blockhash or fee-payer",
+                )),
                 Some(transaction_result) => Ok(transaction_result?),
             })
     }
 
+    /// Send a transaction and return any preflight (sanitization or simulation) errors, or return
+    /// after the transaction has been rejected or reached the given level of commitment.
+    pub fn process_transaction_with_preflight_and_commitment(
+        &mut self,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
+        let mut ctx = context::current();
+        ctx.deadline += Duration::from_secs(50);
+        self.process_transaction_with_preflight_and_commitment_and_context(
+            ctx,
+            transaction,
+            commitment,
+        )
+        .map(|result| match result? {
+            BanksTransactionResultWithSimulation {
+                result: None,
+                simulation_details: _,
+            } => Err(BanksClientError::ClientError(
+                "invalid blockhash or fee-payer",
+            )),
+            BanksTransactionResultWithSimulation {
+                result: Some(Err(err)),
+                simulation_details: Some(simulation_details),
+            } => Err(BanksClientError::SimulationError {
+                err,
+                logs: simulation_details.logs,
+                units_consumed: simulation_details.units_consumed,
+            }),
+            BanksTransactionResultWithSimulation {
+                result: Some(result),
+                simulation_details: _,
+            } => result.map_err(Into::into),
+        })
+    }
+
+    /// Send a transaction and return any preflight (sanitization or simulation) errors, or return
+    /// after the transaction has been finalized or rejected.
+    pub fn process_transaction_with_preflight(
+        &mut self,
+        transaction: Transaction,
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
+        self.process_transaction_with_preflight_and_commitment(
+            transaction,
+            CommitmentLevel::default(),
+        )
+    }
+
     /// Send a transaction and return until the transaction has been finalized or rejected.
     pub fn process_transaction(
         &mut self,
         transaction: Transaction,
-    ) -> impl Future<Output = transport::Result<()>> + '_ {
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
         self.process_transaction_with_commitment(transaction, CommitmentLevel::default())
     }
 
@@ -197,7 +277,7 @@ impl BanksClient {
         &mut self,
         transactions: Vec<Transaction>,
         commitment: CommitmentLevel,
-    ) -> transport::Result<()> {
+    ) -> Result<(), BanksClientError> {
         let mut clients: Vec<_> = transactions.iter().map(|_| self.clone()).collect();
         let futures = clients
             .iter_mut()
@@ -213,19 +293,21 @@ impl BanksClient {
     pub fn process_transactions(
         &mut self,
         transactions: Vec<Transaction>,
-    ) -> impl Future<Output = transport::Result<()>> + '_ {
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
         self.process_transactions_with_commitment(transactions, CommitmentLevel::default())
     }
 
     /// Return the most recent rooted slot. All transactions at or below this slot
     /// are said to be finalized. The cluster will not fork to a higher slot.
-    pub fn get_root_slot(&mut self) -> impl Future<Output = io::Result<Slot>> + '_ {
+    pub fn get_root_slot(&mut self) -> impl Future<Output = Result<Slot, BanksClientError>> + '_ {
         self.get_slot_with_context(context::current(), CommitmentLevel::default())
     }
 
     /// Return the most recent rooted block height. All transactions at or below this height
     /// are said to be finalized. The cluster will not fork to a higher block height.
-    pub fn get_root_block_height(&mut self) -> impl Future<Output = io::Result<Slot>> + '_ {
+    pub fn get_root_block_height(
+        &mut self,
+    ) -> impl Future<Output = Result<Slot, BanksClientError>> + '_ {
         self.get_block_height_with_context(context::current(), CommitmentLevel::default())
     }
 
@@ -235,7 +317,7 @@ impl BanksClient {
         &mut self,
         address: Pubkey,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+    ) -> impl Future<Output = Result<Option<Account>, BanksClientError>> + '_ {
         self.get_account_with_commitment_and_context(context::current(), address, commitment)
     }
 
@@ -244,7 +326,7 @@ impl BanksClient {
     pub fn get_account(
         &mut self,
         address: Pubkey,
-    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+    ) -> impl Future<Output = Result<Option<Account>, BanksClientError>> + '_ {
         self.get_account_with_commitment(address, CommitmentLevel::default())
     }
 
@@ -253,12 +335,11 @@ impl BanksClient {
     pub fn get_packed_account_data<T: Pack>(
         &mut self,
         address: Pubkey,
-    ) -> impl Future<Output = io::Result<T>> + '_ {
+    ) -> impl Future<Output = Result<T, BanksClientError>> + '_ {
         self.get_account(address).map(|result| {
-            let account =
-                result?.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Account not found"))?;
+            let account = result?.ok_or(BanksClientError::ClientError("Account not found"))?;
             T::unpack_from_slice(&account.data)
-                .map_err(|_| io::Error::new(io::ErrorKind::Other, "Failed to deserialize account"))
+                .map_err(|_| BanksClientError::ClientError("Failed to deserialize account"))
         })
     }
 
@@ -267,11 +348,10 @@ impl BanksClient {
     pub fn get_account_data_with_borsh<T: BorshDeserialize>(
         &mut self,
         address: Pubkey,
-    ) -> impl Future<Output = io::Result<T>> + '_ {
+    ) -> impl Future<Output = Result<T, BanksClientError>> + '_ {
         self.get_account(address).map(|result| {
-            let account =
-                result?.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "account not found"))?;
-            T::try_from_slice(&account.data)
+            let account = result?.ok_or(BanksClientError::ClientError("Account not found"))?;
+            T::try_from_slice(&account.data).map_err(Into::into)
         })
     }
 
@@ -281,14 +361,17 @@ impl BanksClient {
         &mut self,
         address: Pubkey,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<u64>> + '_ {
+    ) -> impl Future<Output = Result<u64, BanksClientError>> + '_ {
         self.get_account_with_commitment_and_context(context::current(), address, commitment)
             .map(|result| Ok(result?.map(|x| x.lamports).unwrap_or(0)))
     }
 
     /// Return the balance in lamports of an account at the given address at the time
     /// of the most recent root slot.
-    pub fn get_balance(&mut self, address: Pubkey) -> impl Future<Output = io::Result<u64>> + '_ {
+    pub fn get_balance(
+        &mut self,
+        address: Pubkey,
+    ) -> impl Future<Output = Result<u64, BanksClientError>> + '_ {
         self.get_balance_with_commitment(address, CommitmentLevel::default())
     }
 
@@ -300,7 +383,7 @@ impl BanksClient {
     pub fn get_transaction_status(
         &mut self,
         signature: Signature,
-    ) -> impl Future<Output = io::Result<Option<TransactionStatus>>> + '_ {
+    ) -> impl Future<Output = Result<Option<TransactionStatus>, BanksClientError>> + '_ {
         self.get_transaction_status_with_context(context::current(), signature)
     }
 
@@ -308,7 +391,7 @@ impl BanksClient {
     pub async fn get_transaction_statuses(
         &mut self,
         signatures: Vec<Signature>,
-    ) -> io::Result<Vec<Option<TransactionStatus>>> {
+    ) -> Result<Vec<Option<TransactionStatus>>, BanksClientError> {
         // tarpc futures oddly hold a mutable reference back to the client so clone the client upfront
         let mut clients_and_signatures: Vec<_> = signatures
             .into_iter()
@@ -325,19 +408,22 @@ impl BanksClient {
         statuses.into_iter().collect()
     }
 
-    pub fn get_latest_blockhash(&mut self) -> impl Future<Output = io::Result<Hash>> + '_ {
+    pub fn get_latest_blockhash(
+        &mut self,
+    ) -> impl Future<Output = Result<Hash, BanksClientError>> + '_ {
         self.get_latest_blockhash_with_commitment(CommitmentLevel::default())
             .map(|result| {
                 result?
                     .map(|x| x.0)
-                    .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "account not found"))
+                    .ok_or(BanksClientError::ClientError("valid blockhash not found"))
+                    .map_err(Into::into)
             })
     }
 
     pub fn get_latest_blockhash_with_commitment(
         &mut self,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<Option<(Hash, u64)>>> + '_ {
+    ) -> impl Future<Output = Result<Option<(Hash, u64)>, BanksClientError>> + '_ {
         self.get_latest_blockhash_with_commitment_and_context(context::current(), commitment)
     }
 
@@ -345,9 +431,10 @@ impl BanksClient {
         &mut self,
         ctx: Context,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<Option<(Hash, u64)>>> + '_ {
+    ) -> impl Future<Output = Result<Option<(Hash, u64)>, BanksClientError>> + '_ {
         self.inner
             .get_latest_blockhash_with_commitment_and_context(ctx, commitment)
+            .map_err(Into::into)
     }
 
     pub fn get_fee_for_message_with_commitment_and_context(
@@ -355,13 +442,14 @@ impl BanksClient {
         ctx: Context,
         commitment: CommitmentLevel,
         message: Message,
-    ) -> impl Future<Output = io::Result<Option<u64>>> + '_ {
+    ) -> impl Future<Output = Result<Option<u64>, BanksClientError>> + '_ {
         self.inner
             .get_fee_for_message_with_commitment_and_context(ctx, commitment, message)
+            .map_err(Into::into)
     }
 }
 
-pub async fn start_client<C>(transport: C) -> io::Result<BanksClient>
+pub async fn start_client<C>(transport: C) -> Result<BanksClient, BanksClientError>
 where
     C: Transport<ClientMessage<BanksRequest>, Response<BanksResponse>> + Send + 'static,
 {
@@ -370,7 +458,7 @@ where
     })
 }
 
-pub async fn start_tcp_client<T: ToSocketAddrs>(addr: T) -> io::Result<BanksClient> {
+pub async fn start_tcp_client<T: ToSocketAddrs>(addr: T) -> Result<BanksClient, BanksClientError> {
     let transport = tcp::connect(addr, Bincode::default).await?;
     Ok(BanksClient {
         inner: TarpcClient::new(client::Config::default(), transport).spawn(),
@@ -399,7 +487,7 @@ mod tests {
     }
 
     #[test]
-    fn test_banks_server_transfer_via_server() -> io::Result<()> {
+    fn test_banks_server_transfer_via_server() -> Result<(), BanksClientError> {
         // This test shows the preferred way to interact with BanksServer.
         // It creates a runtime explicitly (no globals via tokio macros) and calls
         // `runtime.block_on()` just once, to run all the async code.
@@ -432,7 +520,7 @@ mod tests {
     }
 
     #[test]
-    fn test_banks_server_transfer_via_client() -> io::Result<()> {
+    fn test_banks_server_transfer_via_client() -> Result<(), BanksClientError> {
         // The caller may not want to hold the connection open until the transaction
         // is processed (or blockhash expires). In this test, we verify the
         // server-side functionality is available to the client.

banks-interface/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-interface"
-version = "1.9.0"
+version = "1.10.0"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,9 +10,9 @@ documentation = "https://docs.rs/solana-banks-interface"
 edition = "2021"
 
 [dependencies]
-serde = { version = "1.0.130", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-tarpc = { version = "0.26.2", features = ["full"] }
+serde = { version = "1.0.133", features = ["derive"] }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+tarpc = { version = "0.27.2", features = ["full"] }
 
 [lib]
 crate-type = ["lib"]

banks-interface/src/lib.rs

@@ -30,6 +30,19 @@ pub struct TransactionStatus {
     pub confirmation_status: Option<TransactionConfirmationStatus>,
 }
 
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TransactionSimulationDetails {
+    pub logs: Vec<String>,
+    pub units_consumed: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct BanksTransactionResultWithSimulation {
+    pub result: Option<transaction::Result<()>>,
+    pub simulation_details: Option<TransactionSimulationDetails>,
+}
+
 #[tarpc::service]
 pub trait Banks {
     async fn send_transaction_with_context(transaction: Transaction);
@@ -44,6 +57,10 @@ pub trait Banks {
         -> Option<TransactionStatus>;
     async fn get_slot_with_context(commitment: CommitmentLevel) -> Slot;
     async fn get_block_height_with_context(commitment: CommitmentLevel) -> u64;
+    async fn process_transaction_with_preflight_and_commitment_and_context(
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> BanksTransactionResultWithSimulation;
     async fn process_transaction_with_commitment_and_context(
         transaction: Transaction,
         commitment: CommitmentLevel,

banks-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-server"
-version = "1.9.0"
+version = "1.10.0"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,12 +11,13 @@ edition = "2021"
 
 [dependencies]
 bincode = "1.3.3"
+crossbeam-channel = "0.5"
 futures = "0.3"
-solana-banks-interface = { path = "../banks-interface", version = "=1.9.0" }
-solana-runtime = { path = "../runtime", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.9.0" }
-tarpc = { version = "0.26.2", features = ["full"] }
+solana-banks-interface = { path = "../banks-interface", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.10.0" }
+tarpc = { version = "0.27.2", features = ["full"] }
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 tokio-stream = "0.1"

banks-server/src/banks_server.rs

@@ -1,10 +1,16 @@
 use {
     bincode::{deserialize, serialize},
+    crossbeam_channel::{unbounded, Receiver, Sender},
     futures::{future, prelude::stream::StreamExt},
     solana_banks_interface::{
-        Banks, BanksRequest, BanksResponse, TransactionConfirmationStatus, TransactionStatus,
+        Banks, BanksRequest, BanksResponse, BanksTransactionResultWithSimulation,
+        TransactionConfirmationStatus, TransactionSimulationDetails, TransactionStatus,
+    },
+    solana_runtime::{
+        bank::{Bank, TransactionSimulationResult},
+        bank_forks::BankForks,
+        commitment::BlockCommitmentCache,
     },
-    solana_runtime::{bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache},
     solana_sdk::{
         account::Account,
         clock::Slot,
@@ -15,7 +21,7 @@ use {
         message::{Message, SanitizedMessage},
         pubkey::Pubkey,
         signature::Signature,
-        transaction::{self, Transaction},
+        transaction::{self, SanitizedTransaction, Transaction},
     },
     solana_send_transaction_service::{
         send_transaction_service::{SendTransactionService, TransactionInfo},
@@ -25,17 +31,14 @@ use {
         convert::TryFrom,
         io,
         net::{Ipv4Addr, SocketAddr},
-        sync::{
-            mpsc::{channel, Receiver, Sender},
-            Arc, RwLock,
-        },
+        sync::{Arc, RwLock},
         thread::Builder,
         time::Duration,
     },
     tarpc::{
         context::Context,
         serde_transport::tcp,
-        server::{self, Channel, Incoming},
+        server::{self, incoming::Incoming, Channel},
         transport::{self, channel::UnboundedChannel},
         ClientMessage, Response,
     },
@@ -91,7 +94,7 @@ impl BanksServer {
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
         poll_signature_status_sleep_duration: Duration,
     ) -> Self {
-        let (transaction_sender, transaction_receiver) = channel();
+        let (transaction_sender, transaction_receiver) = unbounded();
         let bank = bank_forks.read().unwrap().working_bank();
         let slot = bank.slot();
         {
@@ -242,6 +245,47 @@ impl Banks for BanksServer {
         self.bank(commitment).block_height()
     }
 
+    async fn process_transaction_with_preflight_and_commitment_and_context(
+        self,
+        ctx: Context,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> BanksTransactionResultWithSimulation {
+        let sanitized_transaction =
+            match SanitizedTransaction::try_from_legacy_transaction(transaction.clone()) {
+                Err(err) => {
+                    return BanksTransactionResultWithSimulation {
+                        result: Some(Err(err)),
+                        simulation_details: None,
+                    };
+                }
+                Ok(tx) => tx,
+            };
+        if let TransactionSimulationResult {
+            result: Err(err),
+            logs,
+            post_simulation_accounts: _,
+            units_consumed,
+        } = self
+            .bank(commitment)
+            .simulate_transaction_unchecked(sanitized_transaction)
+        {
+            return BanksTransactionResultWithSimulation {
+                result: Some(Err(err)),
+                simulation_details: Some(TransactionSimulationDetails {
+                    logs,
+                    units_consumed,
+                }),
+            };
+        }
+        BanksTransactionResultWithSimulation {
+            result: self
+                .process_transaction_with_commitment_and_context(ctx, transaction, commitment)
+                .await,
+            simulation_details: None,
+        }
+    }
+
     async fn process_transaction_with_commitment_and_context(
         self,
         _: Context,
@@ -346,7 +390,7 @@ pub async fn start_tcp_server(
         // serve is generated by the service attribute. It takes as input any type implementing
         // the generated Banks trait.
         .map(move |chan| {
-            let (sender, receiver) = channel();
+            let (sender, receiver) = unbounded();
 
             SendTransactionService::new::<NullTpuInfo>(
                 tpu_addr,

bench-streamer/Cargo.toml

@@ -2,19 +2,18 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-bench-streamer"
-version = "1.9.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 publish = false
 
 [dependencies]
+crossbeam-channel = "0.5"
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "=1.9.0" }
-solana-streamer = { path = "../streamer", version = "=1.9.0" }
-solana-logger = { path = "../logger", version = "=1.9.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.9.0" }
-solana-version = { path = "../version", version = "=1.9.0" }
+solana-streamer = { path = "../streamer", version = "=1.10.0" }
+solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
+solana-version = { path = "../version", version = "=1.10.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/src/main.rs

@@ -1,16 +1,16 @@
 #![allow(clippy::integer_arithmetic)]
 use {
     clap::{crate_description, crate_name, App, Arg},
+    crossbeam_channel::unbounded,
     solana_streamer::{
-        packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE},
-        streamer::{receiver, PacketReceiver},
+        packet::{Packet, PacketBatch, PacketBatchRecycler, PACKET_DATA_SIZE},
+        streamer::{receiver, PacketBatchReceiver},
     },
     std::{
         cmp::max,
         net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
         sync::{
             atomic::{AtomicBool, AtomicUsize, Ordering},
-            mpsc::channel,
             Arc,
         },
         thread::{sleep, spawn, JoinHandle, Result},
@@ -20,19 +20,19 @@ use {
 
 fn producer(addr: &SocketAddr, exit: Arc<AtomicBool>) -> JoinHandle<()> {
     let send = UdpSocket::bind("0.0.0.0:0").unwrap();
-    let mut msgs = Packets::default();
-    msgs.packets.resize(10, Packet::default());
-    for w in msgs.packets.iter_mut() {
+    let mut packet_batch = PacketBatch::default();
+    packet_batch.packets.resize(10, Packet::default());
+    for w in packet_batch.packets.iter_mut() {
         w.meta.size = PACKET_DATA_SIZE;
         w.meta.set_addr(addr);
     }
-    let msgs = Arc::new(msgs);
+    let packet_batch = Arc::new(packet_batch);
     spawn(move || loop {
         if exit.load(Ordering::Relaxed) {
             return;
         }
         let mut num = 0;
-        for p in &msgs.packets {
+        for p in &packet_batch.packets {
             let a = p.meta.addr();
             assert!(p.meta.size <= PACKET_DATA_SIZE);
             send.send_to(&p.data[..p.meta.size], &a).unwrap();
@@ -42,14 +42,14 @@ fn producer(addr: &SocketAddr, exit: Arc<AtomicBool>) -> JoinHandle<()> {
     })
 }
 
-fn sink(exit: Arc<AtomicBool>, rvs: Arc<AtomicUsize>, r: PacketReceiver) -> JoinHandle<()> {
+fn sink(exit: Arc<AtomicBool>, rvs: Arc<AtomicUsize>, r: PacketBatchReceiver) -> JoinHandle<()> {
     spawn(move || loop {
         if exit.load(Ordering::Relaxed) {
             return;
         }
         let timer = Duration::new(1, 0);
-        if let Ok(msgs) = r.recv_timeout(timer) {
-            rvs.fetch_add(msgs.packets.len(), Ordering::Relaxed);
+        if let Ok(packet_batch) = r.recv_timeout(timer) {
+            rvs.fetch_add(packet_batch.packets.len(), Ordering::Relaxed);
         }
     })
 }
@@ -81,7 +81,7 @@ fn main() -> Result<()> {
 
     let mut read_channels = Vec::new();
     let mut read_threads = Vec::new();
-    let recycler = PacketsRecycler::default();
+    let recycler = PacketBatchRecycler::default();
     for _ in 0..num_sockets {
         let read = solana_net_utils::bind_to(ip_addr, port, false).unwrap();
         read.set_read_timeout(Some(Duration::new(1, 0))).unwrap();
@@ -89,7 +89,7 @@ fn main() -> Result<()> {
         addr = read.local_addr().unwrap();
         port = addr.port();
 
-        let (s_reader, r_reader) = channel();
+        let (s_reader, r_reader) = unbounded();
         read_channels.push(r_reader);
         read_threads.push(receiver(
             Arc::new(read),

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-bench-tps"
-version = "1.9.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,27 +10,28 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
+crossbeam-channel = "0.5"
 log = "0.4.14"
 rayon = "1.5.1"
-serde_json = "1.0.72"
-serde_yaml = "0.8.21"
-solana-core = { path = "../core", version = "=1.9.0" }
-solana-genesis = { path = "../genesis", version = "=1.9.0" }
-solana-client = { path = "../client", version = "=1.9.0" }
-solana-faucet = { path = "../faucet", version = "=1.9.0" }
-solana-gossip = { path = "../gossip", version = "=1.9.0" }
-solana-logger = { path = "../logger", version = "=1.9.0" }
-solana-metrics = { path = "../metrics", version = "=1.9.0" }
-solana-measure = { path = "../measure", version = "=1.9.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.9.0" }
-solana-runtime = { path = "../runtime", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-solana-streamer = { path = "../streamer", version = "=1.9.0" }
-solana-version = { path = "../version", version = "=1.9.0" }
+serde_json = "1.0.74"
+serde_yaml = "0.8.23"
+solana-core = { path = "../core", version = "=1.10.0" }
+solana-genesis = { path = "../genesis", version = "=1.10.0" }
+solana-client = { path = "../client", version = "=1.10.0" }
+solana-faucet = { path = "../faucet", version = "=1.10.0" }
+solana-gossip = { path = "../gossip", version = "=1.10.0" }
+solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-metrics = { path = "../metrics", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-streamer = { path = "../streamer", version = "=1.10.0" }
+solana-version = { path = "../version", version = "=1.10.0" }
 
 [dev-dependencies]
 serial_test = "0.5.1"
-solana-local-cluster = { path = "../local-cluster", version = "=1.9.0" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.10.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/tests/bench_tps.rs

@@ -1,5 +1,6 @@
 #![allow(clippy::integer_arithmetic)]
 use {
+    crossbeam_channel::unbounded,
     serial_test::serial,
     solana_bench_tps::{
         bench::{do_bench_tps, generate_and_fund_keypairs},
@@ -15,10 +16,7 @@ use {
     },
     solana_sdk::signature::{Keypair, Signer},
     solana_streamer::socket::SocketAddrSpace,
-    std::{
-        sync::{mpsc::channel, Arc},
-        time::Duration,
-    },
+    std::{sync::Arc, time::Duration},
 };
 
 fn test_bench_tps_local_cluster(config: Config) {
@@ -52,7 +50,7 @@ fn test_bench_tps_local_cluster(config: Config) {
         VALIDATOR_PORT_RANGE,
     ));
 
-    let (addr_sender, addr_receiver) = channel();
+    let (addr_sender, addr_receiver) = unbounded();
     run_local_faucet_with_port(faucet_keypair, addr_sender, None, 0);
     let faucet_addr = addr_receiver
         .recv_timeout(Duration::from_secs(2))

bucket_map/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bucket-map"
-version = "1.9.0"
+version = "1.10.0"
 description = "solana-bucket-map"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-bucket-map"
@@ -11,15 +11,18 @@ license = "Apache-2.0"
 edition = "2021"
 
 [dependencies]
-rayon = "1.5.0"
-solana-logger = { path = "../logger", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-memmap2 = "0.5.0"
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+memmap2 = "0.5.2"
 log = { version = "0.4.11" }
-solana-measure = { path = "../measure", version = "=1.9.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
 rand = "0.7.0"
+tempfile = "3.3.0"
+modular-bitfield = "0.11.2"
+
+[dev-dependencies]
 fs_extra = "1.2.0"
-tempfile = "3.2.0"
+rayon = "1.5.0"
+solana-logger = { path = "../logger", version = "=1.10.0" }
 
 [lib]
 crate-type = ["lib"]

bucket_map/src/bucket.rs

@@ -3,7 +3,7 @@ use {
         bucket_item::BucketItem,
         bucket_map::BucketMapError,
         bucket_stats::BucketMapStats,
-        bucket_storage::{BucketStorage, Uid, DEFAULT_CAPACITY_POW2, UID_UNLOCKED},
+        bucket_storage::{BucketStorage, Uid, DEFAULT_CAPACITY_POW2},
         index_entry::IndexEntry,
         MaxSearch, RefCount,
     },
@@ -17,7 +17,7 @@ use {
         ops::RangeBounds,
         path::PathBuf,
         sync::{
-            atomic::{AtomicUsize, Ordering},
+            atomic::{AtomicU64, AtomicUsize, Ordering},
             Arc, Mutex,
         },
     },
@@ -81,6 +81,7 @@ impl<T: Clone + Copy> Bucket<T> {
         drives: Arc<Vec<PathBuf>>,
         max_search: MaxSearch,
         stats: Arc<BucketMapStats>,
+        count: Arc<AtomicU64>,
     ) -> Self {
         let index = BucketStorage::new(
             Arc::clone(&drives),
@@ -88,6 +89,7 @@ impl<T: Clone + Copy> Bucket<T> {
             std::mem::size_of::<IndexEntry>() as u64,
             max_search,
             Arc::clone(&stats.index),
+            count,
         );
         Self {
             random: thread_rng().gen(),
@@ -100,14 +102,10 @@ impl<T: Clone + Copy> Bucket<T> {
         }
     }
 
-    pub fn bucket_len(&self) -> u64 {
-        self.index.used.load(Ordering::Relaxed)
-    }
-
     pub fn keys(&self) -> Vec<Pubkey> {
         let mut rv = vec![];
         for i in 0..self.index.capacity() {
-            if self.index.uid(i) == UID_UNLOCKED {
+            if self.index.is_free(i) {
                 continue;
             }
             let ix: &IndexEntry = self.index.get(i);
@@ -120,10 +118,10 @@ impl<T: Clone + Copy> Bucket<T> {
     where
         R: RangeBounds<Pubkey>,
     {
-        let mut result = Vec::with_capacity(self.index.used.load(Ordering::Relaxed) as usize);
+        let mut result = Vec::with_capacity(self.index.count.load(Ordering::Relaxed) as usize);
         for i in 0..self.index.capacity() {
             let ii = i % self.index.capacity();
-            if self.index.uid(ii) == UID_UNLOCKED {
+            if self.index.is_free(ii) {
                 continue;
             }
             let ix: &IndexEntry = self.index.get(ii);
@@ -156,7 +154,7 @@ impl<T: Clone + Copy> Bucket<T> {
         let ix = Self::bucket_index_ix(index, key, random);
         for i in ix..ix + index.max_search() {
             let ii = i % index.capacity();
-            if index.uid(ii) == UID_UNLOCKED {
+            if index.is_free(ii) {
                 continue;
             }
             let elem: &mut IndexEntry = index.get_mut(ii);
@@ -175,7 +173,7 @@ impl<T: Clone + Copy> Bucket<T> {
         let ix = Self::bucket_index_ix(index, key, random);
         for i in ix..ix + index.max_search() {
             let ii = i % index.capacity();
-            if index.uid(ii) == UID_UNLOCKED {
+            if index.is_free(ii) {
                 continue;
             }
             let elem: &IndexEntry = index.get(ii);
@@ -187,26 +185,23 @@ impl<T: Clone + Copy> Bucket<T> {
     }
 
     fn bucket_create_key(
-        index: &BucketStorage,
+        index: &mut BucketStorage,
         key: &Pubkey,
         elem_uid: Uid,
         random: u64,
+        is_resizing: bool,
     ) -> Result<u64, BucketMapError> {
         let ix = Self::bucket_index_ix(index, key, random);
         for i in ix..ix + index.max_search() {
             let ii = i as u64 % index.capacity();
-            if index.uid(ii) != UID_UNLOCKED {
+            if !index.is_free(ii) {
                 continue;
             }
-            index.allocate(ii, elem_uid).unwrap();
-            let mut elem: &mut IndexEntry = index.get_mut(ii);
-            elem.key = *key;
-            // These will be overwritten after allocation by callers.
+            index.allocate(ii, elem_uid, is_resizing).unwrap();
+            let elem: &mut IndexEntry = index.get_mut(ii);
+            // These fields will be overwritten after allocation by callers.
             // Since this part of the mmapped file could have previously been used by someone else, there can be garbage here.
-            elem.ref_count = 0;
-            elem.storage_offset = 0;
-            elem.storage_capacity_when_created_pow2 = 0;
-            elem.num_slots = 0;
+            elem.init(key);
             //debug!(                "INDEX ALLOC {:?} {} {} {}",                key, ii, index.capacity, elem_uid            );
             return Ok(ii);
         }
@@ -225,8 +220,14 @@ impl<T: Clone + Copy> Bucket<T> {
         Some(elem.ref_count)
     }
 
-    fn create_key(&self, key: &Pubkey) -> Result<u64, BucketMapError> {
-        Self::bucket_create_key(&self.index, key, IndexEntry::key_uid(key), self.random)
+    fn create_key(&mut self, key: &Pubkey) -> Result<u64, BucketMapError> {
+        Self::bucket_create_key(
+            &mut self.index,
+            key,
+            IndexEntry::key_uid(key),
+            self.random,
+            false,
+        )
     }
 
     pub fn read_value(&self, key: &Pubkey) -> Option<(&[T], RefCount)> {
@@ -256,16 +257,17 @@ impl<T: Clone + Copy> Bucket<T> {
             Some(res) => res,
         };
         elem.ref_count = ref_count;
-        let elem_uid = self.index.uid(elem_ix);
+        let elem_uid = self.index.uid_unchecked(elem_ix);
         let bucket_ix = elem.data_bucket_ix();
         let current_bucket = &self.data[bucket_ix as usize];
+        let num_slots = data.len() as u64;
         if best_fit_bucket == bucket_ix && elem.num_slots > 0 {
             // in place update
             let elem_loc = elem.data_loc(current_bucket);
             let slice: &mut [T] = current_bucket.get_mut_cell_slice(elem_loc, data.len() as u64);
-            assert!(current_bucket.uid(elem_loc) == elem_uid);
-            elem.num_slots = data.len() as u64;
-            slice.clone_from_slice(data);
+            assert_eq!(current_bucket.uid(elem_loc), Some(elem_uid));
+            elem.num_slots = num_slots;
+            slice.copy_from_slice(data);
             Ok(())
         } else {
             // need to move the allocation to a best fit spot
@@ -275,18 +277,21 @@ impl<T: Clone + Copy> Bucket<T> {
             let pos = thread_rng().gen_range(0, cap);
             for i in pos..pos + self.index.max_search() {
                 let ix = i % cap;
-                if best_bucket.uid(ix) == UID_UNLOCKED {
+                if best_bucket.is_free(ix) {
                     let elem_loc = elem.data_loc(current_bucket);
-                    if elem.num_slots > 0 {
+                    let old_slots = elem.num_slots;
+                    elem.set_storage_offset(ix);
+                    elem.set_storage_capacity_when_created_pow2(best_bucket.capacity_pow2);
+                    elem.num_slots = num_slots;
+                    if old_slots > 0 {
+                        let current_bucket = &mut self.data[bucket_ix as usize];
                         current_bucket.free(elem_loc, elem_uid);
                     }
-                    elem.storage_offset = ix;
-                    elem.storage_capacity_when_created_pow2 = best_bucket.capacity_pow2;
-                    elem.num_slots = data.len() as u64;
                     //debug!(                        "DATA ALLOC {:?} {} {} {}",                        key, elem.data_location, best_bucket.capacity, elem_uid                    );
-                    if elem.num_slots > 0 {
-                        best_bucket.allocate(ix, elem_uid).unwrap();
-                        let slice = best_bucket.get_mut_cell_slice(ix, data.len() as u64);
+                    if num_slots > 0 {
+                        let best_bucket = &mut self.data[best_fit_bucket as usize];
+                        best_bucket.allocate(ix, elem_uid, false).unwrap();
+                        let slice = best_bucket.get_mut_cell_slice(ix, num_slots);
                         slice.copy_from_slice(data);
                     }
                     return Ok(());
@@ -298,10 +303,12 @@ impl<T: Clone + Copy> Bucket<T> {
 
     pub fn delete_key(&mut self, key: &Pubkey) {
         if let Some((elem, elem_ix)) = self.find_entry(key) {
-            let elem_uid = self.index.uid(elem_ix);
+            let elem_uid = self.index.uid_unchecked(elem_ix);
             if elem.num_slots > 0 {
-                let data_bucket = &self.data[elem.data_bucket_ix() as usize];
+                let ix = elem.data_bucket_ix() as usize;
+                let data_bucket = &self.data[ix];
                 let loc = elem.data_loc(data_bucket);
+                let data_bucket = &mut self.data[ix];
                 //debug!(                    "DATA FREE {:?} {} {} {}",                    key, elem.data_location, data_bucket.capacity, elem_uid                );
                 data_bucket.free(loc, elem_uid);
             }
@@ -319,7 +326,7 @@ impl<T: Clone + Copy> Bucket<T> {
                 //increasing the capacity by ^4 reduces the
                 //likelyhood of a re-index collision of 2^(max_search)^2
                 //1 in 2^32
-                let index = BucketStorage::new_with_capacity(
+                let mut index = BucketStorage::new_with_capacity(
                     Arc::clone(&self.drives),
                     1,
                     std::mem::size_of::<IndexEntry>() as u64,
@@ -327,14 +334,16 @@ impl<T: Clone + Copy> Bucket<T> {
                     self.index.capacity_pow2 + i, // * 2,
                     self.index.max_search,
                     Arc::clone(&self.stats.index),
+                    Arc::clone(&self.index.count),
                 );
                 let random = thread_rng().gen();
                 let mut valid = true;
                 for ix in 0..self.index.capacity() {
                     let uid = self.index.uid(ix);
-                    if UID_UNLOCKED != uid {
+                    if let Some(uid) = uid {
                         let elem: &IndexEntry = self.index.get(ix);
-                        let new_ix = Self::bucket_create_key(&index, &elem.key, uid, random);
+                        let new_ix =
+                            Self::bucket_create_key(&mut index, &elem.key, uid, random, true);
                         if new_ix.is_err() {
                             valid = false;
                             break;
@@ -391,6 +400,7 @@ impl<T: Clone + Copy> Bucket<T> {
                     Self::elem_size(),
                     self.index.max_search,
                     Arc::clone(&self.stats.data),
+                    Arc::default(),
                 ))
             }
             self.data.push(bucket);

bucket_map/src/bucket_api.rs

@@ -30,14 +30,13 @@ impl<T: Clone + Copy> BucketApi<T> {
         drives: Arc<Vec<PathBuf>>,
         max_search: MaxSearch,
         stats: Arc<BucketMapStats>,
-        count: Arc<AtomicU64>,
     ) -> Self {
         Self {
             drives,
             max_search,
             stats,
             bucket: RwLock::default(),
-            count,
+            count: Arc::default(),
         }
     }
 
@@ -73,12 +72,7 @@ impl<T: Clone + Copy> BucketApi<T> {
     }
 
     pub fn bucket_len(&self) -> u64 {
-        self.bucket
-            .read()
-            .unwrap()
-            .as_ref()
-            .map(|bucket| bucket.bucket_len())
-            .unwrap_or_default()
+        self.count.load(Ordering::Relaxed)
     }
 
     pub fn delete_key(&self, key: &Pubkey) {
@@ -95,11 +89,11 @@ impl<T: Clone + Copy> BucketApi<T> {
                 Arc::clone(&self.drives),
                 self.max_search,
                 Arc::clone(&self.stats),
+                Arc::clone(&self.count),
             ));
         } else {
             let write = bucket.as_mut().unwrap();
             write.handle_delayed_grows();
-            self.count.store(write.bucket_len(), Ordering::Relaxed);
         }
         bucket
     }

bucket_map/src/bucket_map.rs

@@ -79,21 +79,14 @@ impl<T: Clone + Copy + Debug> BucketMap<T> {
         });
         let drives = Arc::new(drives);
 
-        let mut per_bucket_count = Vec::with_capacity(config.max_buckets);
-        per_bucket_count.resize_with(config.max_buckets, Arc::default);
-        let stats = Arc::new(BucketMapStats {
-            per_bucket_count,
-            ..BucketMapStats::default()
-        });
-        let buckets = stats
-            .per_bucket_count
-            .iter()
-            .map(|per_bucket_count| {
+        let stats = Arc::default();
+        let buckets = (0..config.max_buckets)
+            .into_iter()
+            .map(|_| {
                 Arc::new(BucketApi::new(
                     Arc::clone(&drives),
                     max_search,
                     Arc::clone(&stats),
-                    Arc::clone(per_bucket_count),
                 ))
             })
             .collect();

bucket_map/src/bucket_stats.rs

@@ -14,5 +14,4 @@ pub struct BucketStats {
 pub struct BucketMapStats {
     pub index: Arc<BucketStats>,
     pub data: Arc<BucketStats>,
-    pub per_bucket_count: Vec<Arc<AtomicU64>>,
 }

bucket_map/src/bucket_storage.rs

@@ -35,27 +35,42 @@ use {
 pub const DEFAULT_CAPACITY_POW2: u8 = 5;
 
 /// A Header UID of 0 indicates that the header is unlocked
-pub(crate) const UID_UNLOCKED: Uid = 0;
+const UID_UNLOCKED: Uid = 0;
 
 pub(crate) type Uid = u64;
 
 #[repr(C)]
 struct Header {
-    lock: AtomicU64,
+    lock: u64,
 }
 
 impl Header {
-    fn try_lock(&self, uid: Uid) -> bool {
-        Ok(UID_UNLOCKED)
-            == self
-                .lock
-                .compare_exchange(UID_UNLOCKED, uid, Ordering::AcqRel, Ordering::Relaxed)
+    /// try to lock this entry with 'uid'
+    /// return true if it could be locked
+    fn try_lock(&mut self, uid: Uid) -> bool {
+        if self.lock == UID_UNLOCKED {
+            self.lock = uid;
+            true
+        } else {
+            false
+        }
     }
-    fn unlock(&self) -> Uid {
-        self.lock.swap(UID_UNLOCKED, Ordering::Release)
+    /// mark this entry as unlocked
+    fn unlock(&mut self, expected: Uid) {
+        assert_eq!(expected, self.lock);
+        self.lock = UID_UNLOCKED;
     }
-    fn uid(&self) -> Uid {
-        self.lock.load(Ordering::Acquire)
+    /// uid that has locked this entry or None if unlocked
+    fn uid(&self) -> Option<Uid> {
+        if self.lock == UID_UNLOCKED {
+            None
+        } else {
+            Some(self.lock)
+        }
+    }
+    /// true if this entry is unlocked
+    fn is_unlocked(&self) -> bool {
+        self.lock == UID_UNLOCKED
     }
 }
 
@@ -64,7 +79,7 @@ pub struct BucketStorage {
     mmap: MmapMut,
     pub cell_size: u64,
     pub capacity_pow2: u8,
-    pub used: AtomicU64,
+    pub count: Arc<AtomicU64>,
     pub stats: Arc<BucketStats>,
     pub max_search: MaxSearch,
 }
@@ -88,6 +103,7 @@ impl BucketStorage {
         capacity_pow2: u8,
         max_search: MaxSearch,
         stats: Arc<BucketStats>,
+        count: Arc<AtomicU64>,
     ) -> Self {
         let cell_size = elem_size * num_elems + std::mem::size_of::<Header>() as u64;
         let (mmap, path) = Self::new_map(&drives, cell_size as usize, capacity_pow2, &stats);
@@ -95,7 +111,7 @@ impl BucketStorage {
             path,
             mmap,
             cell_size,
-            used: AtomicU64::new(0),
+            count,
             capacity_pow2,
             stats,
             max_search,
@@ -112,6 +128,7 @@ impl BucketStorage {
         elem_size: u64,
         max_search: MaxSearch,
         stats: Arc<BucketStats>,
+        count: Arc<AtomicU64>,
     ) -> Self {
         Self::new_with_capacity(
             drives,
@@ -120,53 +137,74 @@ impl BucketStorage {
             DEFAULT_CAPACITY_POW2,
             max_search,
             stats,
+            count,
         )
     }
 
-    pub fn uid(&self, ix: u64) -> Uid {
-        assert!(ix < self.capacity(), "bad index size");
+    /// return ref to header of item 'ix' in mmapped file
+    fn header_ptr(&self, ix: u64) -> &Header {
         let ix = (ix * self.cell_size) as usize;
         let hdr_slice: &[u8] = &self.mmap[ix..ix + std::mem::size_of::<Header>()];
         unsafe {
             let hdr = hdr_slice.as_ptr() as *const Header;
-            return hdr.as_ref().unwrap().uid();
+            hdr.as_ref().unwrap()
         }
     }
 
-    pub fn allocate(&self, ix: u64, uid: Uid) -> Result<(), BucketStorageError> {
+    /// return ref to header of item 'ix' in mmapped file
+    fn header_mut_ptr(&mut self, ix: u64) -> &mut Header {
+        let ix = (ix * self.cell_size) as usize;
+        let hdr_slice: &mut [u8] = &mut self.mmap[ix..ix + std::mem::size_of::<Header>()];
+        unsafe {
+            let hdr = hdr_slice.as_mut_ptr() as *mut Header;
+            hdr.as_mut().unwrap()
+        }
+    }
+
+    /// return uid allocated at index 'ix' or None if vacant
+    pub fn uid(&self, ix: u64) -> Option<Uid> {
+        assert!(ix < self.capacity(), "bad index size");
+        self.header_ptr(ix).uid()
+    }
+
+    /// true if the entry at index 'ix' is free (as opposed to being allocated)
+    pub fn is_free(&self, ix: u64) -> bool {
+        // note that the terminology in the implementation is locked or unlocked.
+        // but our api is allocate/free
+        self.header_ptr(ix).is_unlocked()
+    }
+
+    /// caller knows id is not empty
+    pub fn uid_unchecked(&self, ix: u64) -> Uid {
+        self.uid(ix).unwrap()
+    }
+
+    /// 'is_resizing' true if caller is resizing the index (so don't increment count)
+    /// 'is_resizing' false if caller is adding an item to the index (so increment count)
+    pub fn allocate(
+        &mut self,
+        ix: u64,
+        uid: Uid,
+        is_resizing: bool,
+    ) -> Result<(), BucketStorageError> {
         assert!(ix < self.capacity(), "allocate: bad index size");
         assert!(UID_UNLOCKED != uid, "allocate: bad uid");
         let mut e = Err(BucketStorageError::AlreadyAllocated);
-        let ix = (ix * self.cell_size) as usize;
         //debug!("ALLOC {} {}", ix, uid);
-        let hdr_slice: &[u8] = &self.mmap[ix..ix + std::mem::size_of::<Header>()];
-        unsafe {
-            let hdr = hdr_slice.as_ptr() as *const Header;
-            if hdr.as_ref().unwrap().try_lock(uid) {
-                e = Ok(());
-                self.used.fetch_add(1, Ordering::Relaxed);
+        if self.header_mut_ptr(ix).try_lock(uid) {
+            e = Ok(());
+            if !is_resizing {
+                self.count.fetch_add(1, Ordering::Relaxed);
             }
-        };
+        }
         e
     }
 
-    pub fn free(&self, ix: u64, uid: Uid) {
+    pub fn free(&mut self, ix: u64, uid: Uid) {
         assert!(ix < self.capacity(), "bad index size");
         assert!(UID_UNLOCKED != uid, "free: bad uid");
-        let ix = (ix * self.cell_size) as usize;
-        //debug!("FREE {} {}", ix, uid);
-        let hdr_slice: &[u8] = &self.mmap[ix..ix + std::mem::size_of::<Header>()];
-        unsafe {
-            let hdr = hdr_slice.as_ptr() as *const Header;
-            //debug!("FREE uid: {}", hdr.as_ref().unwrap().uid());
-            let previous_uid = hdr.as_ref().unwrap().unlock();
-            assert_eq!(
-                previous_uid, uid,
-                "free: unlocked a header with a differet uid: {}",
-                previous_uid
-            );
-            self.used.fetch_sub(1, Ordering::Relaxed);
-        }
+        self.header_mut_ptr(ix).unlock(uid);
+        self.count.fetch_sub(1, Ordering::Relaxed);
     }
 
     pub fn get<T: Sized>(&self, ix: u64) -> &T {
@@ -324,6 +362,9 @@ impl BucketStorage {
             capacity_pow_2,
             max_search,
             Arc::clone(stats),
+            bucket
+                .map(|bucket| Arc::clone(&bucket.count))
+                .unwrap_or_default(),
         );
         if let Some(bucket) = bucket {
             new_bucket.copy_contents(bucket);
@@ -341,3 +382,43 @@ impl BucketStorage {
         1 << self.capacity_pow2
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_bucket_storage() {
+        let tmpdir1 = std::env::temp_dir().join("bucket_map_test_mt");
+        let paths: Vec<PathBuf> = [tmpdir1]
+            .iter()
+            .filter(|x| std::fs::create_dir_all(x).is_ok())
+            .cloned()
+            .collect();
+        assert!(!paths.is_empty());
+
+        let mut storage =
+            BucketStorage::new(Arc::new(paths), 1, 1, 1, Arc::default(), Arc::default());
+        let ix = 0;
+        let uid = Uid::MAX;
+        assert!(storage.is_free(ix));
+        assert!(storage.allocate(ix, uid, false).is_ok());
+        assert!(storage.allocate(ix, uid, false).is_err());
+        assert!(!storage.is_free(ix));
+        assert_eq!(storage.uid(ix), Some(uid));
+        assert_eq!(storage.uid_unchecked(ix), uid);
+        storage.free(ix, uid);
+        assert!(storage.is_free(ix));
+        assert_eq!(storage.uid(ix), None);
+        let uid = 1;
+        assert!(storage.is_free(ix));
+        assert!(storage.allocate(ix, uid, false).is_ok());
+        assert!(storage.allocate(ix, uid, false).is_err());
+        assert!(!storage.is_free(ix));
+        assert_eq!(storage.uid(ix), Some(uid));
+        assert_eq!(storage.uid_unchecked(ix), uid);
+        storage.free(ix, uid);
+        assert!(storage.is_free(ix));
+        assert_eq!(storage.uid(ix), None);
+    }
+}

bucket_map/src/index_entry.rs

@@ -1,9 +1,11 @@
+#![allow(dead_code)]
 use {
     crate::{
         bucket::Bucket,
         bucket_storage::{BucketStorage, Uid},
         RefCount,
     },
+    modular_bitfield::prelude::*,
     solana_sdk::{clock::Slot, pubkey::Pubkey},
     std::{
         collections::hash_map::DefaultHasher,
@@ -19,13 +21,42 @@ use {
 pub struct IndexEntry {
     pub key: Pubkey, // can this be smaller if we have reduced the keys into buckets already?
     pub ref_count: RefCount, // can this be smaller? Do we ever need more than 4B refcounts?
-    pub storage_offset: u64, // smaller? since these are variably sized, this could get tricky. well, actually accountinfo is not variable sized...
+    storage_cap_and_offset: PackedStorage,
     // if the bucket doubled, the index can be recomputed using create_bucket_capacity_pow2
-    pub storage_capacity_when_created_pow2: u8, // see data_location
     pub num_slots: Slot, // can this be smaller? epoch size should ~ be the max len. this is the num elements in the slot list
 }
 
+/// Pack the storage offset and capacity-when-crated-pow2 fields into a single u64
+#[bitfield(bits = 64)]
+#[repr(C)]
+#[derive(Debug, Default, Copy, Clone, Eq, PartialEq)]
+struct PackedStorage {
+    capacity_when_created_pow2: B8,
+    offset: B56,
+}
+
 impl IndexEntry {
+    pub fn init(&mut self, pubkey: &Pubkey) {
+        self.key = *pubkey;
+        self.ref_count = 0;
+        self.storage_cap_and_offset = PackedStorage::default();
+        self.num_slots = 0;
+    }
+
+    pub fn set_storage_capacity_when_created_pow2(
+        &mut self,
+        storage_capacity_when_created_pow2: u8,
+    ) {
+        self.storage_cap_and_offset
+            .set_capacity_when_created_pow2(storage_capacity_when_created_pow2)
+    }
+
+    pub fn set_storage_offset(&mut self, storage_offset: u64) {
+        self.storage_cap_and_offset
+            .set_offset_checked(storage_offset)
+            .expect("New storage offset must fit into 7 bytes!")
+    }
+
     pub fn data_bucket_from_num_slots(num_slots: Slot) -> u64 {
         (num_slots as f64).log2().ceil() as u64 // use int log here?
     }
@@ -38,10 +69,18 @@ impl IndexEntry {
         self.ref_count
     }
 
+    fn storage_capacity_when_created_pow2(&self) -> u8 {
+        self.storage_cap_and_offset.capacity_when_created_pow2()
+    }
+
+    fn storage_offset(&self) -> u64 {
+        self.storage_cap_and_offset.offset()
+    }
+
     // This function maps the original data location into an index in the current bucket storage.
     // This is coupled with how we resize bucket storages.
     pub fn data_loc(&self, storage: &BucketStorage) -> u64 {
-        self.storage_offset << (storage.capacity_pow2 - self.storage_capacity_when_created_pow2)
+        self.storage_offset() << (storage.capacity_pow2 - self.storage_capacity_when_created_pow2())
     }
 
     pub fn read_value<'a, T>(&self, bucket: &'a Bucket<T>) -> Option<(&'a [T], RefCount)> {
@@ -50,7 +89,7 @@ impl IndexEntry {
         let slice = if self.num_slots > 0 {
             let loc = self.data_loc(data_bucket);
             let uid = Self::key_uid(&self.key);
-            assert_eq!(uid, bucket.data[data_bucket_ix as usize].uid(loc));
+            assert_eq!(Some(uid), bucket.data[data_bucket_ix as usize].uid(loc));
             bucket.data[data_bucket_ix as usize].get_cell_slice(loc, self.num_slots)
         } else {
             // num_slots is 0. This means we don't have an actual allocation.
@@ -59,9 +98,59 @@ impl IndexEntry {
         };
         Some((slice, self.ref_count))
     }
+
     pub fn key_uid(key: &Pubkey) -> Uid {
         let mut s = DefaultHasher::new();
         key.hash(&mut s);
         s.finish().max(1u64)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    impl IndexEntry {
+        pub fn new(key: Pubkey) -> Self {
+            IndexEntry {
+                key,
+                ref_count: 0,
+                storage_cap_and_offset: PackedStorage::default(),
+                num_slots: 0,
+            }
+        }
+    }
+
+    /// verify that accessors for storage_offset and capacity_when_created are
+    /// correct and independent
+    #[test]
+    fn test_api() {
+        for offset in [0, 1, u32::MAX as u64] {
+            let mut index = IndexEntry::new(solana_sdk::pubkey::new_rand());
+            if offset != 0 {
+                index.set_storage_offset(offset);
+            }
+            assert_eq!(index.storage_offset(), offset);
+            assert_eq!(index.storage_capacity_when_created_pow2(), 0);
+            for pow in [1, 255, 0] {
+                index.set_storage_capacity_when_created_pow2(pow);
+                assert_eq!(index.storage_offset(), offset);
+                assert_eq!(index.storage_capacity_when_created_pow2(), pow);
+            }
+        }
+    }
+
+    #[test]
+    fn test_size() {
+        assert_eq!(std::mem::size_of::<PackedStorage>(), 1 + 7);
+        assert_eq!(std::mem::size_of::<IndexEntry>(), 32 + 8 + 8 + 8);
+    }
+
+    #[test]
+    #[should_panic(expected = "New storage offset must fit into 7 bytes!")]
+    fn test_set_storage_offset_value_too_large() {
+        let too_big = 1 << 56;
+        let mut index = IndexEntry::new(Pubkey::new_unique());
+        index.set_storage_offset(too_big);
+    }
+}

cargo-build-bpf

@@ -9,5 +9,8 @@ for a in "$@"; do
   fi
 done
 
-set -x
+set -ex
+if [[ ! -f sdk/bpf/syscalls.txt ]]; then
+  "$here"/cargo build --manifest-path "$here"/programs/bpf_loader/gen-syscall-list/Cargo.toml
+fi
 exec "$here"/cargo run --manifest-path "$here"/sdk/cargo-build-bpf/Cargo.toml -- $maybe_bpf_sdk "$@"

ci/buildkite-pipeline.sh

@@ -226,6 +226,44 @@ EOF
     annotate --style info \
       "downstream-projects skipped as no relevant files were modified"
   fi
+
+  # Downstream Anchor projects backwards compatibility
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-stable-perf.sh \
+             ^ci/test-stable.sh \
+             ^ci/test-local-cluster.sh \
+             ^core/build.rs \
+             ^fetch-perf-libs.sh \
+             ^programs/ \
+             ^sdk/ \
+             ^scripts/build-downstream-anchor-projects.sh \
+      ; then
+    cat >> "$output_file" <<"EOF"
+  - command: "scripts/build-downstream-anchor-projects.sh"
+    name: "downstream-anchor-projects"
+    timeout_in_minutes: 10
+EOF
+  else
+    annotate --style info \
+      "downstream-anchor-projects skipped as no relevant files were modified"
+  fi
+
+  # Wasm support
+  if affects \
+             ^ci/test-wasm.sh \
+             ^ci/test-stable.sh \
+             ^sdk/ \
+      ; then
+    command_step wasm ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-wasm.sh" 20
+  else
+    annotate --style info \
+      "wasm skipped as no relevant files were modified"
+  fi
+
   # Benches...
   if affects \
              .rs$ \
@@ -243,7 +281,15 @@ EOF
 
   command_step "local-cluster" \
     ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster.sh" \
-    50
+    40
+
+  command_step "local-cluster-flakey" \
+    ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster-flakey.sh" \
+    10
+
+  command_step "local-cluster-slow" \
+    ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster-slow.sh" \
+    30
 }
 
 pull_or_push_steps() {

ci/buildkite-secondary.yml

@@ -19,3 +19,8 @@ steps:
     timeout_in_minutes: 240
     name: "publish crate"
     branches: "!master"
+  - command: "ci/publish-tarball.sh"
+    agents:
+      - "queue=release-build-aarch64-apple-darwin"
+    timeout_in_minutes: 60
+    name: "publish tarball (aarch64-apple-darwin)"

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.56.1
+FROM solanalabs/rust:1.57.0
 ARG date
 
 RUN set -x \

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.56.1
+FROM rust:1.57.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0
@@ -11,6 +11,7 @@ RUN set -x \
  && apt-get install apt-transport-https \
  && echo deb https://apt.buildkite.com/buildkite-agent stable main > /etc/apt/sources.list.d/buildkite-agent.list \
  && apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 \
+ && curl -fsSL https://deb.nodesource.com/setup_current.x | bash - \
  && apt update \
  && apt install -y \
       buildkite-agent \
@@ -19,15 +20,20 @@ RUN set -x \
       lcov \
       libudev-dev \
       mscgen \
+      nodejs \
       net-tools \
       rsync \
       sudo \
       golang \
       unzip \
       \
+ && apt remove -y libcurl4-openssl-dev \
  && rm -rf /var/lib/apt/lists/* \
+ && node --version \
+ && npm --version \
  && rustup component add rustfmt \
  && rustup component add clippy \
+ && rustup target add wasm32-unknown-unknown \
  && cargo install cargo-audit \
  && cargo install mdbook \
  && cargo install mdbook-linkcheck \

ci/env.sh

@@ -23,6 +23,9 @@ if [[ -n $CI ]]; then
   elif [[ -n $BUILDKITE ]]; then
     export CI_BRANCH=$BUILDKITE_BRANCH
     export CI_BUILD_ID=$BUILDKITE_BUILD_ID
+    if [[ $BUILDKITE_COMMIT = HEAD ]]; then
+      BUILDKITE_COMMIT="$(git rev-parse HEAD)"
+    fi
     export CI_COMMIT=$BUILDKITE_COMMIT
     export CI_JOB_ID=$BUILDKITE_JOB_ID
     # The standard BUILDKITE_PULL_REQUEST environment variable is always "false" due
@@ -35,7 +38,18 @@ if [[ -n $CI ]]; then
       export CI_BASE_BRANCH=$BUILDKITE_BRANCH
       export CI_PULL_REQUEST=
     fi
-    export CI_OS_NAME=linux
+
+    case "$(uname -s)" in
+    Linux)
+      export CI_OS_NAME=linux
+      ;;
+    Darwin)
+      export CI_OS_NAME=osx
+      ;;
+    *)
+      ;;
+    esac
+
     if [[ -n $BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG ]]; then
       # The solana-secondary pipeline should use the slug of the pipeline that
       # triggered it

ci/publish-tarball.sh

@@ -39,7 +39,11 @@ fi
 
 case "$CI_OS_NAME" in
 osx)
-  TARGET=x86_64-apple-darwin
+  _cputype="$(uname -m)"
+  if [[ $_cputype = arm64 ]]; then
+    _cputype=aarch64
+  fi
+  TARGET=${_cputype}-apple-darwin
   ;;
 linux)
   TARGET=x86_64-unknown-linux-gnu

ci/run-local.sh

@@ -27,6 +27,8 @@ steps+=(test-stable-perf)
 steps+=(test-downstream-builds)
 steps+=(test-bench)
 steps+=(test-local-cluster)
+steps+=(test-local-cluster-flakey)
+steps+=(test-local-cluster-slow)
 
 step_index=0
 if [[ -n "$1" ]]; then

ci/rust-version.sh

@@ -18,13 +18,13 @@
 if [[ -n $RUST_STABLE_VERSION ]]; then
   stable_version="$RUST_STABLE_VERSION"
 else
-  stable_version=1.56.1
+  stable_version=1.57.0
 fi
 
 if [[ -n $RUST_NIGHTLY_VERSION ]]; then
   nightly_version="$RUST_NIGHTLY_VERSION"
 else
-  nightly_version=2021-11-30
+  nightly_version=2021-12-03
 fi
 
 

ci/test-local-cluster-flakey.sh

@@ -0,0 +1 @@
+test-stable.sh

ci/test-local-cluster-slow.sh

@@ -0,0 +1 @@
+test-stable.sh

ci/test-stable.sh

@@ -65,6 +65,9 @@ test-stable-bpf)
     fi
   done
 
+  # bpf-tools version
+  "$cargo_build_bpf" -V
+
   # BPF program instruction count assertion
   bpf_target_path=programs/bpf/target
   _ "$cargo" stable test \
@@ -100,7 +103,30 @@ test-stable-perf)
   ;;
 test-local-cluster)
   _ "$cargo" stable build --release --bins ${V:+--verbose}
-  _ "$cargo" stable test --release --package solana-local-cluster ${V:+--verbose} -- --nocapture --test-threads=1
+  _ "$cargo" stable test --release --package solana-local-cluster --test local_cluster ${V:+--verbose} -- --nocapture --test-threads=1
+  exit 0
+  ;;
+test-local-cluster-flakey)
+  _ "$cargo" stable build --release --bins ${V:+--verbose}
+  _ "$cargo" stable test --release --package solana-local-cluster --test local_cluster_flakey ${V:+--verbose} -- --nocapture --test-threads=1
+  exit 0
+  ;;
+test-local-cluster-slow)
+  _ "$cargo" stable build --release --bins ${V:+--verbose}
+  _ "$cargo" stable test --release --package solana-local-cluster --test local_cluster_slow ${V:+--verbose} -- --nocapture --test-threads=1
+  exit 0
+  ;;
+test-wasm)
+  _ node --version
+  _ npm --version
+  for dir in sdk/{program,}; do
+    if [[ -r "$dir"/package.json ]]; then
+      pushd "$dir"
+      _ npm install
+      _ npm test
+      popd
+    fi
+  done
   exit 0
   ;;
 *)

ci/test-wasm.sh

@@ -0,0 +1 @@
+test-stable.sh

ci/upload-ci-artifact.sh

@@ -19,13 +19,24 @@ upload-ci-artifact() {
 upload-s3-artifact() {
   echo "--- artifact: $1 to $2"
   (
-    set -x
-    docker run \
-      --rm \
-      --env AWS_ACCESS_KEY_ID \
-      --env AWS_SECRET_ACCESS_KEY \
-      --volume "$PWD:/solana" \
-      eremite/aws-cli:2018.12.18 \
+    args=(
+      --rm
+      --env AWS_ACCESS_KEY_ID
+      --env AWS_SECRET_ACCESS_KEY
+      --volume "$PWD:/solana"
+
+    )
+    if [[ $(uname -m) = arm64 ]]; then
+      # Ref: https://blog.jaimyn.dev/how-to-build-multi-architecture-docker-images-on-an-m1-mac/#tldr
+      args+=(
+        --platform linux/amd64
+      )
+    fi
+    args+=(
+      eremite/aws-cli:2018.12.18
       /usr/bin/s3cmd --acl-public put "$1" "$2"
+    )
+    set -x
+    docker run "${args[@]}"
   )
 }

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.9.0"
+version = "1.10.0"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,9 +12,9 @@ edition = "2021"
 [dependencies]
 clap = "2.33.0"
 rpassword = "5.0"
-solana-perf = { path = "../perf", version = "=1.9.0" }
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
+solana-perf = { path = "../perf", version = "=1.10.0" }
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.10.0", default-features = false}
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
 thiserror = "1.0.30"
 tiny-bip39 = "0.8.2"
 uriparse = "0.6.3"
@@ -22,7 +22,7 @@ url = "2.2.2"
 chrono = "0.4"
 
 [dev-dependencies]
-tempfile = "3.2.0"
+tempfile = "3.3.0"
 
 [lib]
 name = "solana_clap_utils"

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.9.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -12,13 +12,13 @@ documentation = "https://docs.rs/solana-cli-config"
 [dependencies]
 dirs-next = "2.0.0"
 lazy_static = "1.4.0"
-serde = "1.0.130"
+serde = "1.0.133"
 serde_derive = "1.0.103"
-serde_yaml = "0.8.21"
+serde_yaml = "0.8.23"
 url = "2.2.2"
 
 [dev-dependencies]
-anyhow = "1.0.51"
+anyhow = "1.0.52"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

cli-output/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.9.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -17,14 +17,14 @@ console = "0.15.0"
 humantime = "2.0.1"
 Inflector = "0.11.4"
 indicatif = "0.16.2"
-serde = "1.0.130"
-serde_json = "1.0.72"
-solana-account-decoder = { path = "../account-decoder", version = "=1.9.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.9.0" }
-solana-client = { path = "../client", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.9.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.9.0" }
+serde = "1.0.133"
+serde_json = "1.0.74"
+solana-account-decoder = { path = "../account-decoder", version = "=1.10.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.10.0" }
+solana-client = { path = "../client", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
+solana-vote-program = { path = "../programs/vote", version = "=1.10.0" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 
 [package.metadata.docs.rs]

cli-output/src/cli_output.rs

@@ -99,7 +99,7 @@ impl OutputFormat {
 pub struct CliAccount {
     #[serde(flatten)]
     pub keyed_account: RpcKeyedAccount,
-    #[serde(skip_serializing)]
+    #[serde(skip_serializing, skip_deserializing)]
     pub use_lamports_unit: bool,
 }
 

cli-output/src/display.rs

@@ -139,7 +139,7 @@ fn format_account_mode(message: &Message, index: usize) -> String {
         } else {
             "-"
         },
-        if message.is_writable(index, /*demote_program_write_locks=*/ true) {
+        if message.is_writable(index) {
             "w" // comment for consistent rust fmt (no joking; lol)
         } else {
             "-"

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2021"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.9.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -17,39 +17,40 @@ criterion-stats = "0.3.0"
 ctrlc = { version = "3.2.1", features = ["termination"] }
 console = "0.15.0"
 const_format = "0.2.22"
+crossbeam-channel = "0.5"
 log = "0.4.14"
 humantime = "2.0.1"
 num-traits = "0.2"
 pretty-hex = "0.2.1"
 reqwest = { version = "0.11.6", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 semver = "1.0.4"
-serde = "1.0.130"
+serde = "1.0.133"
 serde_derive = "1.0.103"
-serde_json = "1.0.72"
-solana-account-decoder = { path = "../account-decoder", version = "=1.9.0" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.9.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.9.0" }
-solana-cli-config = { path = "../cli-config", version = "=1.9.0" }
-solana-cli-output = { path = "../cli-output", version = "=1.9.0" }
-solana-client = { path = "../client", version = "=1.9.0" }
-solana-config-program = { path = "../programs/config", version = "=1.9.0" }
-solana-faucet = { path = "../faucet", version = "=1.9.0" }
-solana-logger = { path = "../logger", version = "=1.9.0" }
-solana-program-runtime = { path = "../program-runtime", version = "=1.9.0" }
-solana_rbpf = "=0.2.16"
-solana-remote-wallet = { path = "../remote-wallet", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.9.0" }
-solana-version = { path = "../version", version = "=1.9.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.9.0" }
+serde_json = "1.0.74"
+solana-account-decoder = { path = "../account-decoder", version = "=1.10.0" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "=1.10.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.10.0" }
+solana-cli-config = { path = "../cli-config", version = "=1.10.0" }
+solana-cli-output = { path = "../cli-output", version = "=1.10.0" }
+solana-client = { path = "../client", version = "=1.10.0" }
+solana-config-program = { path = "../programs/config", version = "=1.10.0" }
+solana-faucet = { path = "../faucet", version = "=1.10.0" }
+solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-program-runtime = { path = "../program-runtime", version = "=1.10.0" }
+solana_rbpf = "=0.2.21"
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
+solana-version = { path = "../version", version = "=1.10.0" }
+solana-vote-program = { path = "../programs/vote", version = "=1.10.0" }
 spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 thiserror = "1.0.30"
 tiny-bip39 = "0.8.2"
 
 [dev-dependencies]
-solana-streamer = { path = "../streamer", version = "=1.9.0" }
-solana-test-validator = { path = "../test-validator", version = "=1.9.0" }
-tempfile = "3.2.0"
+solana-streamer = { path = "../streamer", version = "=1.10.0" }
+solana-test-validator = { path = "../test-validator", version = "=1.10.0" }
+tempfile = "3.3.0"
 
 [[bin]]
 name = "solana"

cli/src/checks.rs

@@ -98,10 +98,7 @@ pub fn get_fee_for_messages(
 ) -> Result<u64, CliError> {
     Ok(messages
         .iter()
-        .map(|message| {
-            println!("msg {:?}", message.recent_blockhash);
-            rpc_client.get_fee_for_message(message)
-        })
+        .map(|message| rpc_client.get_fee_for_message(message))
         .collect::<Result<Vec<_>, _>>()?
         .iter()
         .sum())

cli/src/cli.rs

@@ -298,7 +298,13 @@ pub enum CliCommand {
         authorized_voter: Option<Pubkey>,
         authorized_withdrawer: Pubkey,
         commission: u8,
+        sign_only: bool,
+        dump_transaction_message: bool,
+        blockhash_query: BlockhashQuery,
+        nonce_account: Option<Pubkey>,
+        nonce_authority: SignerIndex,
         memo: Option<String>,
+        fee_payer: SignerIndex,
     },
     ShowVoteAccount {
         pubkey: Pubkey,
@@ -310,19 +316,32 @@ pub enum CliCommand {
         destination_account_pubkey: Pubkey,
         withdraw_authority: SignerIndex,
         withdraw_amount: SpendAmount,
+        sign_only: bool,
+        dump_transaction_message: bool,
+        blockhash_query: BlockhashQuery,
+        nonce_account: Option<Pubkey>,
+        nonce_authority: SignerIndex,
         memo: Option<String>,
+        fee_payer: SignerIndex,
     },
     CloseVoteAccount {
         vote_account_pubkey: Pubkey,
         destination_account_pubkey: Pubkey,
         withdraw_authority: SignerIndex,
         memo: Option<String>,
+        fee_payer: SignerIndex,
     },
     VoteAuthorize {
         vote_account_pubkey: Pubkey,
         new_authorized_pubkey: Pubkey,
         vote_authorize: VoteAuthorize,
+        sign_only: bool,
+        dump_transaction_message: bool,
+        blockhash_query: BlockhashQuery,
+        nonce_account: Option<Pubkey>,
+        nonce_authority: SignerIndex,
         memo: Option<String>,
+        fee_payer: SignerIndex,
         authorized: SignerIndex,
         new_authorized: Option<SignerIndex>,
     },
@@ -330,13 +349,25 @@ pub enum CliCommand {
         vote_account_pubkey: Pubkey,
         new_identity_account: SignerIndex,
         withdraw_authority: SignerIndex,
+        sign_only: bool,
+        dump_transaction_message: bool,
+        blockhash_query: BlockhashQuery,
+        nonce_account: Option<Pubkey>,
+        nonce_authority: SignerIndex,
         memo: Option<String>,
+        fee_payer: SignerIndex,
     },
     VoteUpdateCommission {
         vote_account_pubkey: Pubkey,
         commission: u8,
         withdraw_authority: SignerIndex,
+        sign_only: bool,
+        dump_transaction_message: bool,
+        blockhash_query: BlockhashQuery,
+        nonce_account: Option<Pubkey>,
+        nonce_authority: SignerIndex,
         memo: Option<String>,
+        fee_payer: SignerIndex,
     },
     // Wallet Commands
     Address,
@@ -1384,7 +1415,13 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             authorized_voter,
             authorized_withdrawer,
             commission,
+            sign_only,
+            dump_transaction_message,
+            blockhash_query,
+            ref nonce_account,
+            nonce_authority,
             memo,
+            fee_payer,
         } => process_create_vote_account(
             &rpc_client,
             config,
@@ -1394,7 +1431,13 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             authorized_voter,
             *authorized_withdrawer,
             *commission,
+            *sign_only,
+            *dump_transaction_message,
+            blockhash_query,
+            nonce_account.as_ref(),
+            *nonce_authority,
             memo.as_ref(),
+            *fee_payer,
         ),
         CliCommand::ShowVoteAccount {
             pubkey: vote_account_pubkey,
@@ -1412,7 +1455,13 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             withdraw_authority,
             withdraw_amount,
             destination_account_pubkey,
+            sign_only,
+            dump_transaction_message,
+            blockhash_query,
+            ref nonce_account,
+            nonce_authority,
             memo,
+            fee_payer,
         } => process_withdraw_from_vote_account(
             &rpc_client,
             config,
@@ -1420,13 +1469,20 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             *withdraw_authority,
             *withdraw_amount,
             destination_account_pubkey,
+            *sign_only,
+            *dump_transaction_message,
+            blockhash_query,
+            nonce_account.as_ref(),
+            *nonce_authority,
             memo.as_ref(),
+            *fee_payer,
         ),
         CliCommand::CloseVoteAccount {
             vote_account_pubkey,
             withdraw_authority,
             destination_account_pubkey,
             memo,
+            fee_payer,
         } => process_close_vote_account(
             &rpc_client,
             config,
@@ -1434,12 +1490,19 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             *withdraw_authority,
             destination_account_pubkey,
             memo.as_ref(),
+            *fee_payer,
         ),
         CliCommand::VoteAuthorize {
             vote_account_pubkey,
             new_authorized_pubkey,
             vote_authorize,
+            sign_only,
+            dump_transaction_message,
+            blockhash_query,
+            nonce_account,
+            nonce_authority,
             memo,
+            fee_payer,
             authorized,
             new_authorized,
         } => process_vote_authorize(
@@ -1450,33 +1513,63 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             *vote_authorize,
             *authorized,
             *new_authorized,
+            *sign_only,
+            *dump_transaction_message,
+            blockhash_query,
+            *nonce_account,
+            *nonce_authority,
             memo.as_ref(),
+            *fee_payer,
         ),
         CliCommand::VoteUpdateValidator {
             vote_account_pubkey,
             new_identity_account,
             withdraw_authority,
+            sign_only,
+            dump_transaction_message,
+            blockhash_query,
+            nonce_account,
+            nonce_authority,
             memo,
+            fee_payer,
         } => process_vote_update_validator(
             &rpc_client,
             config,
             vote_account_pubkey,
             *new_identity_account,
             *withdraw_authority,
+            *sign_only,
+            *dump_transaction_message,
+            blockhash_query,
+            *nonce_account,
+            *nonce_authority,
             memo.as_ref(),
+            *fee_payer,
         ),
         CliCommand::VoteUpdateCommission {
             vote_account_pubkey,
             commission,
             withdraw_authority,
+            sign_only,
+            dump_transaction_message,
+            blockhash_query,
+            nonce_account,
+            nonce_authority,
             memo,
+            fee_payer,
         } => process_vote_update_commission(
             &rpc_client,
             config,
             vote_account_pubkey,
             *commission,
             *withdraw_authority,
+            *sign_only,
+            *dump_transaction_message,
+            blockhash_query,
+            *nonce_account,
+            *nonce_authority,
             memo.as_ref(),
+            *fee_payer,
         ),
 
         // Wallet Commands
@@ -1975,7 +2068,13 @@ mod tests {
             authorized_voter: Some(bob_pubkey),
             authorized_withdrawer: bob_pubkey,
             commission: 0,
+            sign_only: false,
+            dump_transaction_message: false,
+            blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+            nonce_account: None,
+            nonce_authority: 0,
             memo: None,
+            fee_payer: 0,
         };
         config.signers = vec![&keypair, &bob_keypair, &identity_keypair];
         let result = process_command(&config);
@@ -2006,7 +2105,13 @@ mod tests {
             vote_account_pubkey: bob_pubkey,
             new_authorized_pubkey,
             vote_authorize: VoteAuthorize::Withdrawer,
+            sign_only: false,
+            dump_transaction_message: false,
+            blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+            nonce_account: None,
+            nonce_authority: 0,
             memo: None,
+            fee_payer: 0,
             authorized: 0,
             new_authorized: None,
         };
@@ -2019,7 +2124,13 @@ mod tests {
             vote_account_pubkey: bob_pubkey,
             new_identity_account: 2,
             withdraw_authority: 1,
+            sign_only: false,
+            dump_transaction_message: false,
+            blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+            nonce_account: None,
+            nonce_authority: 0,
             memo: None,
+            fee_payer: 0,
         };
         let result = process_command(&config);
         assert!(result.is_ok());
@@ -2195,7 +2306,13 @@ mod tests {
             authorized_voter: Some(bob_pubkey),
             authorized_withdrawer: bob_pubkey,
             commission: 0,
+            sign_only: false,
+            dump_transaction_message: false,
+            blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+            nonce_account: None,
+            nonce_authority: 0,
             memo: None,
+            fee_payer: 0,
         };
         config.signers = vec![&keypair, &bob_keypair, &identity_keypair];
         assert!(process_command(&config).is_err());
@@ -2204,7 +2321,13 @@ mod tests {
             vote_account_pubkey: bob_pubkey,
             new_authorized_pubkey: bob_pubkey,
             vote_authorize: VoteAuthorize::Voter,
+            sign_only: false,
+            dump_transaction_message: false,
+            blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+            nonce_account: None,
+            nonce_authority: 0,
             memo: None,
+            fee_payer: 0,
             authorized: 0,
             new_authorized: None,
         };
@@ -2214,7 +2337,13 @@ mod tests {
             vote_account_pubkey: bob_pubkey,
             new_identity_account: 1,
             withdraw_authority: 1,
+            sign_only: false,
+            dump_transaction_message: false,
+            blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+            nonce_account: None,
+            nonce_authority: 0,
             memo: None,
+            fee_payer: 0,
         };
         assert!(process_command(&config).is_err());
 

cli/src/cluster_query.rs

@@ -5,6 +5,7 @@ use {
     },
     clap::{value_t, value_t_or_exit, App, AppSettings, Arg, ArgMatches, SubCommand},
     console::{style, Emoji},
+    crossbeam_channel::unbounded,
     serde::{Deserialize, Serialize},
     solana_clap_utils::{
         input_parsers::*,
@@ -1368,7 +1369,7 @@ pub fn process_ping(
     println_name_value("Source Account:", &config.signers[0].pubkey().to_string());
     println!();
 
-    let (signal_sender, signal_receiver) = std::sync::mpsc::channel();
+    let (signal_sender, signal_receiver) = unbounded();
     ctrlc::set_handler(move || {
         let _ = signal_sender.send(());
     })
@@ -2128,7 +2129,7 @@ pub fn process_calculate_rent(
         timing::years_as_slots(1.0, &seconds_per_tick, clock::DEFAULT_TICKS_PER_SLOT);
     let slots_per_epoch = epoch_schedule.slots_per_epoch as f64;
     let years_per_epoch = slots_per_epoch / slots_per_year;
-    let (lamports_per_epoch, _) = rent.due(0, data_length, years_per_epoch);
+    let lamports_per_epoch = rent.due(0, data_length, years_per_epoch).lamports();
     let cli_rent_calculation = CliRentCalculation {
         lamports_per_byte_year: rent.lamports_per_byte_year,
         lamports_per_epoch,

cli/src/feature.rs

@@ -5,7 +5,7 @@ use {
     },
     clap::{App, AppSettings, Arg, ArgMatches, SubCommand},
     console::style,
-    serde::{Deserialize, Serialize},
+    serde::{Deserialize, Deserializer, Serialize, Serializer},
     solana_clap_utils::{input_parsers::*, input_validators::*, keypair::*},
     solana_cli_output::{QuietDisplay, VerboseDisplay},
     solana_client::{client_error::ClientError, rpc_client::RpcClient},
@@ -23,6 +23,7 @@ use {
         cmp::Ordering,
         collections::{HashMap, HashSet},
         fmt,
+        str::FromStr,
         sync::Arc,
     },
 };
@@ -45,7 +46,7 @@ pub enum FeatureCliCommand {
     },
 }
 
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, PartialEq, Eq)]
 #[serde(rename_all = "camelCase", tag = "status", content = "sinceSlot")]
 pub enum CliFeatureStatus {
     Inactive,
@@ -53,7 +54,29 @@ pub enum CliFeatureStatus {
     Active(Slot),
 }
 
-#[derive(Serialize, Deserialize)]
+impl PartialOrd for CliFeatureStatus {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for CliFeatureStatus {
+    fn cmp(&self, other: &Self) -> Ordering {
+        match (self, other) {
+            (Self::Inactive, Self::Inactive) => Ordering::Equal,
+            (Self::Inactive, _) => Ordering::Greater,
+            (_, Self::Inactive) => Ordering::Less,
+            (Self::Pending, Self::Pending) => Ordering::Equal,
+            (Self::Pending, _) => Ordering::Greater,
+            (_, Self::Pending) => Ordering::Less,
+            (Self::Active(self_active_slot), Self::Active(other_active_slot)) => {
+                self_active_slot.cmp(other_active_slot)
+            }
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, PartialEq, Eq)]
 #[serde(rename_all = "camelCase")]
 pub struct CliFeature {
     pub id: String,
@@ -62,11 +85,28 @@ pub struct CliFeature {
     pub status: CliFeatureStatus,
 }
 
+impl PartialOrd for CliFeature {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl Ord for CliFeature {
+    fn cmp(&self, other: &Self) -> Ordering {
+        match self.status.cmp(&other.status) {
+            Ordering::Equal => self.id.cmp(&other.id),
+            ordering => ordering,
+        }
+    }
+}
+
 #[derive(Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct CliFeatures {
     pub features: Vec<CliFeature>,
     pub feature_activation_allowed: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cluster_feature_sets: Option<CliClusterFeatureSets>,
     #[serde(skip)]
     pub inactive: bool,
 }
@@ -93,11 +133,16 @@ impl fmt::Display for CliFeatures {
                     CliFeatureStatus::Inactive => style("inactive".to_string()).red(),
                     CliFeatureStatus::Pending => style("activation pending".to_string()).yellow(),
                     CliFeatureStatus::Active(activation_slot) =>
-                        style(format!("active since slot {}", activation_slot)).green(),
+                        style(format!("active since slot {:>9}", activation_slot)).green(),
                 },
                 feature.description,
             )?;
         }
+
+        if let Some(feature_sets) = &self.cluster_feature_sets {
+            write!(f, "{}", feature_sets)?;
+        }
+
         if self.inactive && !self.feature_activation_allowed {
             writeln!(
                 f,
@@ -114,6 +159,191 @@ impl fmt::Display for CliFeatures {
 impl QuietDisplay for CliFeatures {}
 impl VerboseDisplay for CliFeatures {}
 
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliClusterFeatureSets {
+    pub tool_feature_set: u32,
+    pub feature_sets: Vec<CliFeatureSet>,
+    #[serde(skip)]
+    pub stake_allowed: bool,
+    #[serde(skip)]
+    pub rpc_allowed: bool,
+}
+
+impl fmt::Display for CliClusterFeatureSets {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let mut tool_feature_set_matches_cluster = false;
+
+        let software_versions_title = "Software Version";
+        let feature_set_title = "Feature Set";
+        let stake_percent_title = "Stake";
+        let rpc_percent_title = "RPC";
+        let mut max_software_versions_len = software_versions_title.len();
+        let mut max_feature_set_len = feature_set_title.len();
+        let mut max_stake_percent_len = stake_percent_title.len();
+        let mut max_rpc_percent_len = rpc_percent_title.len();
+
+        let feature_sets: Vec<_> = self
+            .feature_sets
+            .iter()
+            .map(|feature_set_info| {
+                let me = if self.tool_feature_set == feature_set_info.feature_set {
+                    tool_feature_set_matches_cluster = true;
+                    true
+                } else {
+                    false
+                };
+                let software_versions: Vec<_> = feature_set_info
+                    .software_versions
+                    .iter()
+                    .map(ToString::to_string)
+                    .collect();
+                let software_versions = software_versions.join(", ");
+                let feature_set = if feature_set_info.feature_set == 0 {
+                    "unknown".to_string()
+                } else {
+                    feature_set_info.feature_set.to_string()
+                };
+                let stake_percent = format!("{:.2}%", feature_set_info.stake_percent);
+                let rpc_percent = format!("{:.2}%", feature_set_info.rpc_percent);
+
+                max_software_versions_len = max_software_versions_len.max(software_versions.len());
+                max_feature_set_len = max_feature_set_len.max(feature_set.len());
+                max_stake_percent_len = max_stake_percent_len.max(stake_percent.len());
+                max_rpc_percent_len = max_rpc_percent_len.max(rpc_percent.len());
+
+                (
+                    software_versions,
+                    feature_set,
+                    stake_percent,
+                    rpc_percent,
+                    me,
+                )
+            })
+            .collect();
+
+        if !tool_feature_set_matches_cluster {
+            writeln!(
+                f,
+                "\n{}",
+                style("To activate features the tool and cluster feature sets must match, select a tool version that matches the cluster")
+                    .bold())?;
+        } else {
+            if !self.stake_allowed {
+                write!(
+                    f,
+                    "\n{}",
+                    style("To activate features the stake must be >= 95%")
+                        .bold()
+                        .red()
+                )?;
+            }
+            if !self.rpc_allowed {
+                write!(
+                    f,
+                    "\n{}",
+                    style("To activate features the RPC nodes must be >= 95%")
+                        .bold()
+                        .red()
+                )?;
+            }
+        }
+        writeln!(
+            f,
+            "\n\n{}",
+            style(format!("Tool Feature Set: {}", self.tool_feature_set)).bold()
+        )?;
+        writeln!(
+            f,
+            "{}",
+            style(format!(
+                "{1:<0$}  {3:<2$}  {5:<4$}  {7:<6$}",
+                max_software_versions_len,
+                software_versions_title,
+                max_feature_set_len,
+                feature_set_title,
+                max_stake_percent_len,
+                stake_percent_title,
+                max_rpc_percent_len,
+                rpc_percent_title,
+            ))
+            .bold(),
+        )?;
+        for (software_versions, feature_set, stake_percent, rpc_percent, me) in feature_sets {
+            writeln!(
+                f,
+                "{1:<0$}  {3:>2$}  {5:>4$} {7:>6$}  {8}",
+                max_software_versions_len,
+                software_versions,
+                max_feature_set_len,
+                feature_set,
+                max_stake_percent_len,
+                stake_percent,
+                max_rpc_percent_len,
+                rpc_percent,
+                if me { "<-- me" } else { "" },
+            )?;
+        }
+        writeln!(f)
+    }
+}
+
+impl QuietDisplay for CliClusterFeatureSets {}
+impl VerboseDisplay for CliClusterFeatureSets {}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliFeatureSet {
+    software_versions: Vec<CliVersion>,
+    feature_set: u32,
+    stake_percent: f64,
+    rpc_percent: f32,
+}
+
+#[derive(Eq, PartialEq, Ord, PartialOrd)]
+struct CliVersion(Option<semver::Version>);
+
+impl fmt::Display for CliVersion {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let s = match &self.0 {
+            None => "unknown".to_string(),
+            Some(version) => version.to_string(),
+        };
+        write!(f, "{}", s)
+    }
+}
+
+impl FromStr for CliVersion {
+    type Err = semver::Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let version_option = if s == "unknown" {
+            None
+        } else {
+            Some(semver::Version::from_str(s)?)
+        };
+        Ok(CliVersion(version_option))
+    }
+}
+
+impl Serialize for CliVersion {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&self.to_string())
+    }
+}
+
+impl<'de> Deserialize<'de> for CliVersion {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s: &str = Deserialize::deserialize(deserializer)?;
+        CliVersion::from_str(s).map_err(serde::de::Error::custom)
+    }
+}
+
 pub trait FeatureSubCommands {
     fn feature_subcommands(self) -> Self;
 }
@@ -330,7 +560,10 @@ fn feature_set_stats(rpc_client: &RpcClient) -> Result<FeatureSetStats, ClientEr
 }
 
 // Feature activation is only allowed when 95% of the active stake is on the current feature set
-fn feature_activation_allowed(rpc_client: &RpcClient, quiet: bool) -> Result<bool, ClientError> {
+fn feature_activation_allowed(
+    rpc_client: &RpcClient,
+    quiet: bool,
+) -> Result<(bool, Option<CliClusterFeatureSets>), ClientError> {
     let my_feature_set = solana_version::Version::default().feature_set;
 
     let feature_set_stats = feature_set_stats(rpc_client)?;
@@ -346,54 +579,43 @@ fn feature_activation_allowed(rpc_client: &RpcClient, quiet: bool) -> Result<boo
         )
         .unwrap_or((false, false));
 
-    if !quiet {
-        if feature_set_stats.get(&my_feature_set).is_none() {
-            println!(
-                "{}",
-                style("To activate features the tool and cluster feature sets must match, select a tool version that matches the cluster")
-                    .bold());
-        } else {
-            if !stake_allowed {
-                print!(
-                    "\n{}",
-                    style("To activate features the stake must be >= 95%")
-                        .bold()
-                        .red()
-                );
-            }
-            if !rpc_allowed {
-                print!(
-                    "\n{}",
-                    style("To activate features the RPC nodes must be >= 95%")
-                        .bold()
-                        .red()
-                );
-            }
-        }
-        println!(
-            "\n\n{}",
-            style(format!("Tool Feature Set: {}", my_feature_set)).bold()
-        );
-
-        let mut feature_set_stats = feature_set_stats.into_iter().collect::<Vec<_>>();
-        feature_set_stats.sort_by(|l, r| {
-            match l.1.software_versions[0]
-                .cmp(&r.1.software_versions[0])
+    let cluster_feature_sets = if quiet {
+        None
+    } else {
+        let mut feature_sets = feature_set_stats
+            .into_iter()
+            .map(
+                |(
+                    feature_set,
+                    FeatureSetStatsEntry {
+                        stake_percent,
+                        rpc_nodes_percent: rpc_percent,
+                        software_versions,
+                    },
+                )| {
+                    CliFeatureSet {
+                        software_versions: software_versions.into_iter().map(CliVersion).collect(),
+                        feature_set,
+                        stake_percent,
+                        rpc_percent,
+                    }
+                },
+            )
+            .collect::<Vec<_>>();
+        feature_sets.sort_by(|l, r| {
+            match l.software_versions[0]
+                .cmp(&r.software_versions[0])
                 .reverse()
             {
                 Ordering::Equal => {
                     match l
-                        .1
                         .stake_percent
-                        .partial_cmp(&r.1.stake_percent)
+                        .partial_cmp(&r.stake_percent)
                         .unwrap()
                         .reverse()
                     {
                         Ordering::Equal => {
-                            l.1.rpc_nodes_percent
-                                .partial_cmp(&r.1.rpc_nodes_percent)
-                                .unwrap()
-                                .reverse()
+                            l.rpc_percent.partial_cmp(&r.rpc_percent).unwrap().reverse()
                         }
                         o => o,
                     }
@@ -401,96 +623,15 @@ fn feature_activation_allowed(rpc_client: &RpcClient, quiet: bool) -> Result<boo
                 o => o,
             }
         });
+        Some(CliClusterFeatureSets {
+            tool_feature_set: my_feature_set,
+            feature_sets,
+            stake_allowed,
+            rpc_allowed,
+        })
+    };
 
-        let software_versions_title = "Software Version";
-        let feature_set_title = "Feature Set";
-        let stake_percent_title = "Stake";
-        let rpc_percent_title = "RPC";
-        let mut stats_output = Vec::new();
-        let mut max_software_versions_len = software_versions_title.len();
-        let mut max_feature_set_len = feature_set_title.len();
-        let mut max_stake_percent_len = stake_percent_title.len();
-        let mut max_rpc_percent_len = rpc_percent_title.len();
-        for (
-            feature_set,
-            FeatureSetStatsEntry {
-                stake_percent,
-                rpc_nodes_percent,
-                software_versions,
-            },
-        ) in feature_set_stats.into_iter()
-        {
-            let me = feature_set == my_feature_set;
-            let feature_set = if feature_set == 0 {
-                "unknown".to_string()
-            } else {
-                feature_set.to_string()
-            };
-            let stake_percent = format!("{:.2}%", stake_percent);
-            let rpc_percent = format!("{:.2}%", rpc_nodes_percent);
-
-            let mut has_unknown = false;
-            let mut software_versions = software_versions
-                .iter()
-                .filter_map(|v| {
-                    if v.is_none() {
-                        has_unknown = true;
-                    }
-                    v.as_ref()
-                })
-                .map(ToString::to_string)
-                .collect::<Vec<_>>();
-            if has_unknown {
-                software_versions.push("unknown".to_string());
-            }
-            let software_versions = software_versions.join(", ");
-            max_software_versions_len = max_software_versions_len.max(software_versions.len());
-
-            max_feature_set_len = max_feature_set_len.max(feature_set.len());
-            max_stake_percent_len = max_stake_percent_len.max(stake_percent.len());
-            max_rpc_percent_len = max_rpc_percent_len.max(rpc_percent.len());
-
-            stats_output.push((
-                software_versions,
-                feature_set,
-                stake_percent,
-                rpc_percent,
-                me,
-            ));
-        }
-        println!(
-            "{}",
-            style(format!(
-                "{1:<0$}  {3:<2$}  {5:<4$}  {7:<6$}",
-                max_software_versions_len,
-                software_versions_title,
-                max_feature_set_len,
-                feature_set_title,
-                max_stake_percent_len,
-                stake_percent_title,
-                max_rpc_percent_len,
-                rpc_percent_title,
-            ))
-            .bold(),
-        );
-        for (software_versions, feature_set, stake_percent, rpc_percent, me) in stats_output {
-            println!(
-                "{1:<0$}  {3:>2$}  {5:>4$} {7:>6$}  {8}",
-                max_software_versions_len,
-                software_versions,
-                max_feature_set_len,
-                feature_set,
-                max_stake_percent_len,
-                stake_percent,
-                max_rpc_percent_len,
-                rpc_percent,
-                if me { "<-- me" } else { "" },
-            );
-        }
-        println!();
-    }
-
-    Ok(stake_allowed && rpc_allowed)
+    Ok((stake_allowed && rpc_allowed, cluster_feature_sets))
 }
 
 fn status_from_account(account: Account) -> Option<CliFeatureStatus> {
@@ -550,10 +691,14 @@ fn process_status(
         });
     }
 
-    let feature_activation_allowed = feature_activation_allowed(rpc_client, features.len() <= 1)?;
+    features.sort_unstable();
+
+    let (feature_activation_allowed, cluster_feature_sets) =
+        feature_activation_allowed(rpc_client, features.len() <= 1)?;
     let feature_set = CliFeatures {
         features,
         feature_activation_allowed,
+        cluster_feature_sets,
         inactive,
     };
     Ok(config.output_format.formatted_string(&feature_set))
@@ -577,7 +722,7 @@ fn process_activate(
         }
     }
 
-    if !feature_activation_allowed(rpc_client, false)? {
+    if !feature_activation_allowed(rpc_client, false)?.0 {
         match force {
         ForceActivation::Almost =>
             return Err("Add force argument once more to override the sanity check to force feature activation ".into()),

cli/src/nonce.rs

@@ -334,9 +334,17 @@ pub fn check_nonce_account(
     match state_from_account(nonce_account)? {
         State::Initialized(ref data) => {
             if &data.blockhash != nonce_hash {
-                Err(Error::InvalidHash.into())
+                Err(Error::InvalidHash {
+                    provided: *nonce_hash,
+                    expected: data.blockhash,
+                }
+                .into())
             } else if nonce_authority != &data.authority {
-                Err(Error::InvalidAuthority.into())
+                Err(Error::InvalidAuthority {
+                    provided: *nonce_authority,
+                    expected: data.authority,
+                }
+                .into())
             } else {
                 Ok(())
             }
@@ -946,15 +954,22 @@ mod tests {
             hash(b"invalid"),
             0,
         )));
-        let invalid_hash = Account::new_data(1, &data, &system_program::ID);
+        let invalid_hash = Account::new_data(1, &data, &system_program::ID).unwrap();
         if let CliError::InvalidNonce(err) =
-            check_nonce_account(&invalid_hash.unwrap(), &nonce_pubkey, &blockhash).unwrap_err()
+            check_nonce_account(&invalid_hash, &nonce_pubkey, &blockhash).unwrap_err()
         {
-            assert_eq!(err, Error::InvalidHash,);
+            assert_eq!(
+                err,
+                Error::InvalidHash {
+                    provided: blockhash,
+                    expected: hash(b"invalid"),
+                }
+            );
         }
 
+        let new_nonce_authority = solana_sdk::pubkey::new_rand();
         let data = Versions::new_current(State::Initialized(nonce::state::Data::new(
-            solana_sdk::pubkey::new_rand(),
+            new_nonce_authority,
             blockhash,
             0,
         )));
@@ -962,7 +977,13 @@ mod tests {
         if let CliError::InvalidNonce(err) =
             check_nonce_account(&invalid_authority.unwrap(), &nonce_pubkey, &blockhash).unwrap_err()
         {
-            assert_eq!(err, Error::InvalidAuthority,);
+            assert_eq!(
+                err,
+                Error::InvalidAuthority {
+                    provided: nonce_pubkey,
+                    expected: new_nonce_authority,
+                }
+            );
         }
 
         let data = Versions::new_current(State::Uninitialized);

cli/src/program.rs

@@ -42,6 +42,7 @@ use {
         system_instruction::{self, SystemError},
         system_program,
         transaction::{Transaction, TransactionError},
+        transaction_context::TransactionContext,
     },
     std::{
         fs::File,
@@ -1990,7 +1991,8 @@ fn read_and_verify_elf(program_location: &str) -> Result<Vec<u8>, Box<dyn std::e
     let mut program_data = Vec::new();
     file.read_to_end(&mut program_data)
         .map_err(|err| format!("Unable to read program file: {}", err))?;
-    let mut invoke_context = InvokeContext::new_mock(&[], &[]);
+    let mut transaction_context = TransactionContext::new(Vec::new(), 1);
+    let mut invoke_context = InvokeContext::new_mock(&mut transaction_context, &[]);
 
     // Verify the program
     Executable::<BpfError, ThisInstructionMeter>::from_elf(

cli/src/spend_utils.rs

@@ -16,6 +16,7 @@ use {
 pub enum SpendAmount {
     All,
     Some(u64),
+    RentExempt,
 }
 
 impl Default for SpendAmount {
@@ -90,6 +91,7 @@ where
             0,
             from_pubkey,
             fee_pubkey,
+            0,
             build_message,
         )?;
         Ok((message, spend))
@@ -97,6 +99,12 @@ where
         let from_balance = rpc_client
             .get_balance_with_commitment(from_pubkey, commitment)?
             .value;
+        let from_rent_exempt_minimum = if amount == SpendAmount::RentExempt {
+            let data = rpc_client.get_account_data(from_pubkey)?;
+            rpc_client.get_minimum_balance_for_rent_exemption(data.len())?
+        } else {
+            0
+        };
         let (message, SpendAndFee { spend, fee }) = resolve_spend_message(
             rpc_client,
             amount,
@@ -104,6 +112,7 @@ where
             from_balance,
             from_pubkey,
             fee_pubkey,
+            from_rent_exempt_minimum,
             build_message,
         )?;
         if from_pubkey == fee_pubkey {
@@ -140,6 +149,7 @@ fn resolve_spend_message<F>(
     from_balance: u64,
     from_pubkey: &Pubkey,
     fee_pubkey: &Pubkey,
+    from_rent_exempt_minimum: u64,
     build_message: F,
 ) -> Result<(Message, SpendAndFee), CliError>
 where
@@ -176,5 +186,20 @@ where
                 },
             ))
         }
+        SpendAmount::RentExempt => {
+            let mut lamports = if from_pubkey == fee_pubkey {
+                from_balance.saturating_sub(fee)
+            } else {
+                from_balance
+            };
+            lamports = lamports.saturating_sub(from_rent_exempt_minimum);
+            Ok((
+                build_message(lamports),
+                SpendAndFee {
+                    spend: lamports,
+                    fee,
+                },
+            ))
+        }
     }
 }

cli/src/test_utils.rs

@@ -1,23 +1,29 @@
 use {
     solana_client::rpc_client::RpcClient,
-    solana_sdk::{clock::DEFAULT_MS_PER_SLOT, commitment_config::CommitmentConfig, pubkey::Pubkey},
+    solana_sdk::{clock::DEFAULT_MS_PER_SLOT, commitment_config::CommitmentConfig},
     std::{thread::sleep, time::Duration},
 };
 
-pub fn check_recent_balance(expected_balance: u64, client: &RpcClient, pubkey: &Pubkey) {
-    (0..5).for_each(|tries| {
-        let balance = client
-            .get_balance_with_commitment(pubkey, CommitmentConfig::processed())
-            .unwrap()
-            .value;
-        if balance == expected_balance {
-            return;
-        }
-        if tries == 4 {
-            assert_eq!(balance, expected_balance);
-        }
-        sleep(Duration::from_millis(500));
-    });
+#[macro_export]
+macro_rules! check_balance {
+    ($expected_balance:expr, $client:expr, $pubkey:expr) => {
+        (0..5).for_each(|tries| {
+            let balance = $client
+                .get_balance_with_commitment($pubkey, CommitmentConfig::processed())
+                .unwrap()
+                .value;
+            if balance == $expected_balance {
+                return;
+            }
+            if tries == 4 {
+                assert_eq!(balance, $expected_balance);
+            }
+            std::thread::sleep(std::time::Duration::from_millis(500));
+        });
+    };
+    ($expected_balance:expr, $client:expr, $pubkey:expr,) => {
+        check_balance!($expected_balance, $client, $pubkey)
+    };
 }
 
 pub fn check_ready(rpc_client: &RpcClient) {

cli/src/wallet.rs

@@ -39,7 +39,7 @@ use {
         system_program,
         transaction::Transaction,
     },
-    solana_transaction_status::{EncodedTransaction, UiTransactionEncoding},
+    solana_transaction_status::{Encodable, EncodedTransaction, UiTransactionEncoding},
     std::{fmt::Write as FmtWrite, fs::File, io::Write, sync::Arc},
 };
 
@@ -462,18 +462,27 @@ pub fn process_show_account(
 
     let mut account_string = config.output_format.formatted_string(&cli_account);
 
-    if config.output_format == OutputFormat::Display
-        || config.output_format == OutputFormat::DisplayVerbose
-    {
-        if let Some(output_file) = output_file {
-            let mut f = File::create(output_file)?;
-            f.write_all(&data)?;
-            writeln!(&mut account_string)?;
-            writeln!(&mut account_string, "Wrote account data to {}", output_file)?;
-        } else if !data.is_empty() {
-            use pretty_hex::*;
-            writeln!(&mut account_string, "{:?}", data.hex_dump())?;
+    match config.output_format {
+        OutputFormat::Json | OutputFormat::JsonCompact => {
+            if let Some(output_file) = output_file {
+                let mut f = File::create(output_file)?;
+                f.write_all(account_string.as_bytes())?;
+                writeln!(&mut account_string)?;
+                writeln!(&mut account_string, "Wrote account to {}", output_file)?;
+            }
         }
+        OutputFormat::Display | OutputFormat::DisplayVerbose => {
+            if let Some(output_file) = output_file {
+                let mut f = File::create(output_file)?;
+                f.write_all(&data)?;
+                writeln!(&mut account_string)?;
+                writeln!(&mut account_string, "Wrote account data to {}", output_file)?;
+            } else if !data.is_empty() {
+                use pretty_hex::*;
+                writeln!(&mut account_string, "{:?}", data.hex_dump())?;
+            }
+        }
+        OutputFormat::DisplayQuiet => (),
     }
 
     Ok(account_string)
@@ -555,10 +564,8 @@ pub fn process_confirm(
                                 .transaction
                                 .decode()
                                 .expect("Successful decode");
-                            let json_transaction = EncodedTransaction::encode(
-                                decoded_transaction.clone(),
-                                UiTransactionEncoding::Json,
-                            );
+                            let json_transaction =
+                                decoded_transaction.encode(UiTransactionEncoding::Json);
 
                             transaction = Some(CliTransaction {
                                 transaction: json_transaction,
@@ -600,7 +607,7 @@ pub fn process_decode_transaction(config: &CliConfig, transaction: &Transaction)
     let sigverify_status = CliSignatureVerificationStatus::verify_transaction(transaction);
     let decode_transaction = CliTransaction {
         decoded_transaction: transaction.clone(),
-        transaction: EncodedTransaction::encode(transaction.clone(), UiTransactionEncoding::Json),
+        transaction: transaction.encode(UiTransactionEncoding::Json),
         meta: None,
         block_time: None,
         slot: None,

cli/tests/nonce.rs

@@ -1,8 +1,10 @@
+#![allow(clippy::integer_arithmetic)]
 use {
     solana_cli::{
+        check_balance,
         cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig},
         spend_utils::SpendAmount,
-        test_utils::{check_ready, check_recent_balance},
+        test_utils::check_ready,
     },
     solana_cli_output::{parse_sign_only_reply_string, OutputFormat},
     solana_client::{
@@ -14,6 +16,7 @@ use {
     solana_sdk::{
         commitment_config::CommitmentConfig,
         hash::Hash,
+        native_token::sol_to_lamports,
         pubkey::Pubkey,
         signature::{keypair_from_seed, Keypair, Signer},
         system_program,
@@ -73,10 +76,14 @@ fn full_battery_tests(
         &rpc_client,
         &config_payer,
         &config_payer.signers[0].pubkey(),
-        2000,
+        sol_to_lamports(2000.0),
     )
     .unwrap();
-    check_recent_balance(2000, &rpc_client, &config_payer.signers[0].pubkey());
+    check_balance!(
+        sol_to_lamports(2000.0),
+        &rpc_client,
+        &config_payer.signers[0].pubkey(),
+    );
 
     let mut config_nonce = CliConfig::recent_for_tests();
     config_nonce.json_rpc_url = json_rpc_url;
@@ -108,12 +115,16 @@ fn full_battery_tests(
         seed,
         nonce_authority: optional_authority,
         memo: None,
-        amount: SpendAmount::Some(1000),
+        amount: SpendAmount::Some(sol_to_lamports(1000.0)),
     };
 
     process_command(&config_payer).unwrap();
-    check_recent_balance(1000, &rpc_client, &config_payer.signers[0].pubkey());
-    check_recent_balance(1000, &rpc_client, &nonce_account);
+    check_balance!(
+        sol_to_lamports(1000.0),
+        &rpc_client,
+        &config_payer.signers[0].pubkey(),
+    );
+    check_balance!(sol_to_lamports(1000.0), &rpc_client, &nonce_account);
 
     // Get nonce
     config_payer.signers.pop();
@@ -161,12 +172,16 @@ fn full_battery_tests(
         nonce_authority: index,
         memo: None,
         destination_account_pubkey: payee_pubkey,
-        lamports: 100,
+        lamports: sol_to_lamports(100.0),
     };
     process_command(&config_payer).unwrap();
-    check_recent_balance(1000, &rpc_client, &config_payer.signers[0].pubkey());
-    check_recent_balance(900, &rpc_client, &nonce_account);
-    check_recent_balance(100, &rpc_client, &payee_pubkey);
+    check_balance!(
+        sol_to_lamports(1000.0),
+        &rpc_client,
+        &config_payer.signers[0].pubkey(),
+    );
+    check_balance!(sol_to_lamports(900.0), &rpc_client, &nonce_account);
+    check_balance!(sol_to_lamports(100.0), &rpc_client, &payee_pubkey);
 
     // Show nonce account
     config_payer.command = CliCommand::ShowNonceAccount {
@@ -208,12 +223,16 @@ fn full_battery_tests(
         nonce_authority: 1,
         memo: None,
         destination_account_pubkey: payee_pubkey,
-        lamports: 100,
+        lamports: sol_to_lamports(100.0),
     };
     process_command(&config_payer).unwrap();
-    check_recent_balance(1000, &rpc_client, &config_payer.signers[0].pubkey());
-    check_recent_balance(800, &rpc_client, &nonce_account);
-    check_recent_balance(200, &rpc_client, &payee_pubkey);
+    check_balance!(
+        sol_to_lamports(1000.0),
+        &rpc_client,
+        &config_payer.signers[0].pubkey(),
+    );
+    check_balance!(sol_to_lamports(800.0), &rpc_client, &nonce_account);
+    check_balance!(sol_to_lamports(200.0), &rpc_client, &payee_pubkey);
 }
 
 #[test]
@@ -241,19 +260,27 @@ fn test_create_account_with_seed() {
         &rpc_client,
         &CliConfig::recent_for_tests(),
         &offline_nonce_authority_signer.pubkey(),
-        42,
+        sol_to_lamports(42.0),
     )
     .unwrap();
     request_and_confirm_airdrop(
         &rpc_client,
         &CliConfig::recent_for_tests(),
         &online_nonce_creator_signer.pubkey(),
-        4242,
+        sol_to_lamports(4242.0),
     )
     .unwrap();
-    check_recent_balance(42, &rpc_client, &offline_nonce_authority_signer.pubkey());
-    check_recent_balance(4242, &rpc_client, &online_nonce_creator_signer.pubkey());
-    check_recent_balance(0, &rpc_client, &to_address);
+    check_balance!(
+        sol_to_lamports(42.0),
+        &rpc_client,
+        &offline_nonce_authority_signer.pubkey(),
+    );
+    check_balance!(
+        sol_to_lamports(4242.0),
+        &rpc_client,
+        &online_nonce_creator_signer.pubkey(),
+    );
+    check_balance!(0, &rpc_client, &to_address);
 
     check_ready(&rpc_client);
 
@@ -263,7 +290,7 @@ fn test_create_account_with_seed() {
     let seed = authority_pubkey.to_string()[0..32].to_string();
     let nonce_address =
         Pubkey::create_with_seed(&creator_pubkey, &seed, &system_program::id()).unwrap();
-    check_recent_balance(0, &rpc_client, &nonce_address);
+    check_balance!(0, &rpc_client, &nonce_address);
 
     let mut creator_config = CliConfig::recent_for_tests();
     creator_config.json_rpc_url = test_validator.rpc_url();
@@ -273,13 +300,21 @@ fn test_create_account_with_seed() {
         seed: Some(seed),
         nonce_authority: Some(authority_pubkey),
         memo: None,
-        amount: SpendAmount::Some(241),
+        amount: SpendAmount::Some(sol_to_lamports(241.0)),
     };
     process_command(&creator_config).unwrap();
-    check_recent_balance(241, &rpc_client, &nonce_address);
-    check_recent_balance(42, &rpc_client, &offline_nonce_authority_signer.pubkey());
-    check_recent_balance(4000, &rpc_client, &online_nonce_creator_signer.pubkey());
-    check_recent_balance(0, &rpc_client, &to_address);
+    check_balance!(sol_to_lamports(241.0), &rpc_client, &nonce_address);
+    check_balance!(
+        sol_to_lamports(42.0),
+        &rpc_client,
+        &offline_nonce_authority_signer.pubkey(),
+    );
+    check_balance!(
+        sol_to_lamports(4000.999999999),
+        &rpc_client,
+        &online_nonce_creator_signer.pubkey(),
+    );
+    check_balance!(0, &rpc_client, &to_address);
 
     // Fetch nonce hash
     let nonce_hash = nonce_utils::get_account_with_commitment(
@@ -299,7 +334,7 @@ fn test_create_account_with_seed() {
     authority_config.command = CliCommand::ClusterVersion;
     process_command(&authority_config).unwrap_err();
     authority_config.command = CliCommand::Transfer {
-        amount: SpendAmount::Some(10),
+        amount: SpendAmount::Some(sol_to_lamports(10.0)),
         to: to_address,
         from: 0,
         sign_only: true,
@@ -325,7 +360,7 @@ fn test_create_account_with_seed() {
     submit_config.json_rpc_url = test_validator.rpc_url();
     submit_config.signers = vec![&authority_presigner];
     submit_config.command = CliCommand::Transfer {
-        amount: SpendAmount::Some(10),
+        amount: SpendAmount::Some(sol_to_lamports(10.0)),
         to: to_address,
         from: 0,
         sign_only: false,
@@ -344,8 +379,16 @@ fn test_create_account_with_seed() {
         derived_address_program_id: None,
     };
     process_command(&submit_config).unwrap();
-    check_recent_balance(241, &rpc_client, &nonce_address);
-    check_recent_balance(31, &rpc_client, &offline_nonce_authority_signer.pubkey());
-    check_recent_balance(4000, &rpc_client, &online_nonce_creator_signer.pubkey());
-    check_recent_balance(10, &rpc_client, &to_address);
+    check_balance!(sol_to_lamports(241.0), &rpc_client, &nonce_address);
+    check_balance!(
+        sol_to_lamports(31.999999999),
+        &rpc_client,
+        &offline_nonce_authority_signer.pubkey(),
+    );
+    check_balance!(
+        sol_to_lamports(4000.999999999),
+        &rpc_client,
+        &online_nonce_creator_signer.pubkey(),
+    );
+    check_balance!(sol_to_lamports(10.0), &rpc_client, &to_address);
 }

cli/tests/program.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 use {
     serde_json::Value,
     solana_cli::{

cli/tests/request_airdrop.rs

@@ -1,9 +1,11 @@
+#![allow(clippy::integer_arithmetic)]
 use {
     solana_cli::cli::{process_command, CliCommand, CliConfig},
     solana_client::rpc_client::RpcClient,
     solana_faucet::faucet::run_local_faucet,
     solana_sdk::{
         commitment_config::CommitmentConfig,
+        native_token::sol_to_lamports,
         signature::{Keypair, Signer},
     },
     solana_streamer::socket::SocketAddrSpace,
@@ -22,7 +24,7 @@ fn test_cli_request_airdrop() {
     bob_config.json_rpc_url = test_validator.rpc_url();
     bob_config.command = CliCommand::Airdrop {
         pubkey: None,
-        lamports: 50,
+        lamports: sol_to_lamports(50.0),
     };
     let keypair = Keypair::new();
     bob_config.signers = vec![&keypair];
@@ -36,5 +38,5 @@ fn test_cli_request_airdrop() {
     let balance = rpc_client
         .get_balance(&bob_config.signers[0].pubkey())
         .unwrap();
-    assert_eq!(balance, 50);
+    assert_eq!(balance, sol_to_lamports(50.0));
 }

cli/tests/stake.rs

@@ -1,10 +1,12 @@
+#![allow(clippy::integer_arithmetic)]
 #![allow(clippy::redundant_closure)]
 use {
     solana_cli::{
+        check_balance,
         cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig},
         spend_utils::SpendAmount,
         stake::StakeAuthorizationIndexed,
-        test_utils::{check_ready, check_recent_balance},
+        test_utils::check_ready,
     },
     solana_cli_output::{parse_sign_only_reply_string, OutputFormat},
     solana_client::{
@@ -59,7 +61,13 @@ fn test_stake_delegation_force() {
         authorized_voter: None,
         authorized_withdrawer,
         commission: 0,
+        sign_only: false,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+        nonce_account: None,
+        nonce_authority: 0,
         memo: None,
+        fee_payer: 0,
     };
     process_command(&config).unwrap();
 
@@ -144,7 +152,7 @@ fn test_seed_stake_delegation_and_deactivation() {
         100_000,
     )
     .unwrap();
-    check_recent_balance(100_000, &rpc_client, &config_validator.signers[0].pubkey());
+    check_balance!(100_000, &rpc_client, &config_validator.signers[0].pubkey());
 
     let stake_address = Pubkey::create_with_seed(
         &config_validator.signers[0].pubkey(),
@@ -233,7 +241,7 @@ fn test_stake_delegation_and_deactivation() {
         100_000,
     )
     .unwrap();
-    check_recent_balance(100_000, &rpc_client, &config_validator.signers[0].pubkey());
+    check_balance!(100_000, &rpc_client, &config_validator.signers[0].pubkey());
 
     // Create stake account
     config_validator.signers.push(&stake_keypair);
@@ -327,7 +335,7 @@ fn test_offline_stake_delegation_and_deactivation() {
         100_000,
     )
     .unwrap();
-    check_recent_balance(100_000, &rpc_client, &config_validator.signers[0].pubkey());
+    check_balance!(100_000, &rpc_client, &config_validator.signers[0].pubkey());
 
     request_and_confirm_airdrop(
         &rpc_client,
@@ -336,7 +344,7 @@ fn test_offline_stake_delegation_and_deactivation() {
         100_000,
     )
     .unwrap();
-    check_recent_balance(100_000, &rpc_client, &config_offline.signers[0].pubkey());
+    check_balance!(100_000, &rpc_client, &config_offline.signers[0].pubkey());
 
     // Create stake account
     config_validator.signers.push(&stake_keypair);
@@ -905,13 +913,13 @@ fn test_stake_authorize_with_fee_payer() {
     process_command(&config_offline).unwrap_err();
 
     request_and_confirm_airdrop(&rpc_client, &config, &default_pubkey, 100_000).unwrap();
-    check_recent_balance(100_000, &rpc_client, &config.signers[0].pubkey());
+    check_balance!(100_000, &rpc_client, &config.signers[0].pubkey());
 
     request_and_confirm_airdrop(&rpc_client, &config_payer, &payer_pubkey, 100_000).unwrap();
-    check_recent_balance(100_000, &rpc_client, &payer_pubkey);
+    check_balance!(100_000, &rpc_client, &payer_pubkey);
 
     request_and_confirm_airdrop(&rpc_client, &config_offline, &offline_pubkey, 100_000).unwrap();
-    check_recent_balance(100_000, &rpc_client, &offline_pubkey);
+    check_balance!(100_000, &rpc_client, &offline_pubkey);
 
     check_ready(&rpc_client);
 
@@ -938,7 +946,7 @@ fn test_stake_authorize_with_fee_payer() {
     };
     process_command(&config).unwrap();
     // `config` balance should be 50,000 - 1 stake account sig - 1 fee sig
-    check_recent_balance(50_000 - SIG_FEE - SIG_FEE, &rpc_client, &default_pubkey);
+    check_balance!(50_000 - SIG_FEE - SIG_FEE, &rpc_client, &default_pubkey);
 
     // Assign authority with separate fee payer
     config.signers = vec![&default_signer, &payer_keypair];
@@ -962,10 +970,10 @@ fn test_stake_authorize_with_fee_payer() {
     };
     process_command(&config).unwrap();
     // `config` balance has not changed, despite submitting the TX
-    check_recent_balance(50_000 - SIG_FEE - SIG_FEE, &rpc_client, &default_pubkey);
+    check_balance!(50_000 - SIG_FEE - SIG_FEE, &rpc_client, &default_pubkey);
     // `config_payer` however has paid `config`'s authority sig
     // and `config_payer`'s fee sig
-    check_recent_balance(100_000 - SIG_FEE - SIG_FEE, &rpc_client, &payer_pubkey);
+    check_balance!(100_000 - SIG_FEE - SIG_FEE, &rpc_client, &payer_pubkey);
 
     // Assign authority with offline fee payer
     let blockhash = rpc_client.get_latest_blockhash().unwrap();
@@ -1013,10 +1021,10 @@ fn test_stake_authorize_with_fee_payer() {
     };
     process_command(&config).unwrap();
     // `config`'s balance again has not changed
-    check_recent_balance(50_000 - SIG_FEE - SIG_FEE, &rpc_client, &default_pubkey);
+    check_balance!(50_000 - SIG_FEE - SIG_FEE, &rpc_client, &default_pubkey);
     // `config_offline` however has paid 1 sig due to being both authority
     // and fee payer
-    check_recent_balance(100_000 - SIG_FEE, &rpc_client, &offline_pubkey);
+    check_balance!(100_000 - SIG_FEE, &rpc_client, &offline_pubkey);
 }
 
 #[test]
@@ -1052,10 +1060,10 @@ fn test_stake_split() {
 
     request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 500_000)
         .unwrap();
-    check_recent_balance(500_000, &rpc_client, &config.signers[0].pubkey());
+    check_balance!(500_000, &rpc_client, &config.signers[0].pubkey());
 
     request_and_confirm_airdrop(&rpc_client, &config_offline, &offline_pubkey, 100_000).unwrap();
-    check_recent_balance(100_000, &rpc_client, &offline_pubkey);
+    check_balance!(100_000, &rpc_client, &offline_pubkey);
 
     // Create stake account, identity is authority
     let minimum_stake_balance = rpc_client
@@ -1082,7 +1090,7 @@ fn test_stake_split() {
         from: 0,
     };
     process_command(&config).unwrap();
-    check_recent_balance(
+    check_balance!(
         10 * minimum_stake_balance,
         &rpc_client,
         &stake_account_pubkey,
@@ -1102,7 +1110,7 @@ fn test_stake_split() {
         amount: SpendAmount::Some(minimum_nonce_balance),
     };
     process_command(&config).unwrap();
-    check_recent_balance(minimum_nonce_balance, &rpc_client, &nonce_account.pubkey());
+    check_balance!(minimum_nonce_balance, &rpc_client, &nonce_account.pubkey());
 
     // Fetch nonce hash
     let nonce_hash = nonce_utils::get_account_with_commitment(
@@ -1116,7 +1124,7 @@ fn test_stake_split() {
 
     // Nonced offline split
     let split_account = keypair_from_seed(&[2u8; 32]).unwrap();
-    check_recent_balance(0, &rpc_client, &split_account.pubkey());
+    check_balance!(0, &rpc_client, &split_account.pubkey());
     config_offline.signers.push(&split_account);
     config_offline.command = CliCommand::SplitStake {
         stake_account_pubkey,
@@ -1156,12 +1164,12 @@ fn test_stake_split() {
         fee_payer: 0,
     };
     process_command(&config).unwrap();
-    check_recent_balance(
+    check_balance!(
         8 * minimum_stake_balance,
         &rpc_client,
         &stake_account_pubkey,
     );
-    check_recent_balance(
+    check_balance!(
         2 * minimum_stake_balance,
         &rpc_client,
         &split_account.pubkey(),
@@ -1201,10 +1209,10 @@ fn test_stake_set_lockup() {
 
     request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 500_000)
         .unwrap();
-    check_recent_balance(500_000, &rpc_client, &config.signers[0].pubkey());
+    check_balance!(500_000, &rpc_client, &config.signers[0].pubkey());
 
     request_and_confirm_airdrop(&rpc_client, &config_offline, &offline_pubkey, 100_000).unwrap();
-    check_recent_balance(100_000, &rpc_client, &offline_pubkey);
+    check_balance!(100_000, &rpc_client, &offline_pubkey);
 
     // Create stake account, identity is authority
     let minimum_stake_balance = rpc_client
@@ -1238,7 +1246,12 @@ fn test_stake_set_lockup() {
         from: 0,
     };
     process_command(&config).unwrap();
-    check_recent_balance(
+    check_balance!(
+        10 * minimum_stake_balance,
+        &rpc_client,
+        &stake_account_pubkey,
+    );
+    check_balance!(
         10 * minimum_stake_balance,
         &rpc_client,
         &stake_account_pubkey,
@@ -1371,7 +1384,7 @@ fn test_stake_set_lockup() {
         amount: SpendAmount::Some(minimum_nonce_balance),
     };
     process_command(&config).unwrap();
-    check_recent_balance(minimum_nonce_balance, &rpc_client, &nonce_account_pubkey);
+    check_balance!(minimum_nonce_balance, &rpc_client, &nonce_account_pubkey);
 
     // Fetch nonce hash
     let nonce_hash = nonce_utils::get_account_with_commitment(
@@ -1467,10 +1480,10 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
 
     request_and_confirm_airdrop(&rpc_client, &config, &config.signers[0].pubkey(), 200_000)
         .unwrap();
-    check_recent_balance(200_000, &rpc_client, &config.signers[0].pubkey());
+    check_balance!(200_000, &rpc_client, &config.signers[0].pubkey());
 
     request_and_confirm_airdrop(&rpc_client, &config_offline, &offline_pubkey, 100_000).unwrap();
-    check_recent_balance(100_000, &rpc_client, &offline_pubkey);
+    check_balance!(100_000, &rpc_client, &offline_pubkey);
 
     // Create nonce account
     let minimum_nonce_balance = rpc_client
@@ -1547,7 +1560,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         from: 0,
     };
     process_command(&config).unwrap();
-    check_recent_balance(50_000, &rpc_client, &stake_pubkey);
+    check_balance!(50_000, &rpc_client, &stake_pubkey);
 
     // Fetch nonce hash
     let nonce_hash = nonce_utils::get_account_with_commitment(
@@ -1566,7 +1579,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     config_offline.command = CliCommand::WithdrawStake {
         stake_account_pubkey: stake_pubkey,
         destination_account_pubkey: recipient_pubkey,
-        amount: SpendAmount::Some(42),
+        amount: SpendAmount::Some(50_000),
         withdraw_authority: 0,
         custodian: None,
         sign_only: true,
@@ -1585,7 +1598,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     config.command = CliCommand::WithdrawStake {
         stake_account_pubkey: stake_pubkey,
         destination_account_pubkey: recipient_pubkey,
-        amount: SpendAmount::Some(42),
+        amount: SpendAmount::Some(50_000),
         withdraw_authority: 0,
         custodian: None,
         sign_only: false,
@@ -1601,7 +1614,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         fee_payer: 0,
     };
     process_command(&config).unwrap();
-    check_recent_balance(42, &rpc_client, &recipient_pubkey);
+    check_balance!(50_000, &rpc_client, &recipient_pubkey);
 
     // Fetch nonce hash
     let nonce_hash = nonce_utils::get_account_with_commitment(
@@ -1661,7 +1674,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     process_command(&config).unwrap();
     let seed_address =
         Pubkey::create_with_seed(&stake_pubkey, seed, &stake::program::id()).unwrap();
-    check_recent_balance(50_000, &rpc_client, &seed_address);
+    check_balance!(50_000, &rpc_client, &seed_address);
 }
 
 #[test]

cli/tests/transfer.rs

@@ -1,9 +1,11 @@
+#![allow(clippy::integer_arithmetic)]
 #![allow(clippy::redundant_closure)]
 use {
     solana_cli::{
+        check_balance,
         cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig},
         spend_utils::SpendAmount,
-        test_utils::{check_ready, check_recent_balance},
+        test_utils::check_ready,
     },
     solana_cli_output::{parse_sign_only_reply_string, OutputFormat},
     solana_client::{
@@ -14,6 +16,7 @@ use {
     solana_faucet::faucet::run_local_faucet,
     solana_sdk::{
         commitment_config::CommitmentConfig,
+        native_token::sol_to_lamports,
         nonce::State as NonceState,
         pubkey::Pubkey,
         signature::{keypair_from_seed, Keypair, NullSigner, Signer},
@@ -49,15 +52,16 @@ fn test_transfer() {
     let sender_pubkey = config.signers[0].pubkey();
     let recipient_pubkey = Pubkey::new(&[1u8; 32]);
 
-    request_and_confirm_airdrop(&rpc_client, &config, &sender_pubkey, 50_000).unwrap();
-    check_recent_balance(50_000, &rpc_client, &sender_pubkey);
-    check_recent_balance(0, &rpc_client, &recipient_pubkey);
+    request_and_confirm_airdrop(&rpc_client, &config, &sender_pubkey, sol_to_lamports(5.0))
+        .unwrap();
+    check_balance!(sol_to_lamports(5.0), &rpc_client, &sender_pubkey);
+    check_balance!(0, &rpc_client, &recipient_pubkey);
 
     check_ready(&rpc_client);
 
     // Plain ole transfer
     config.command = CliCommand::Transfer {
-        amount: SpendAmount::Some(10),
+        amount: SpendAmount::Some(sol_to_lamports(1.0)),
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
@@ -73,12 +77,12 @@ fn test_transfer() {
         derived_address_program_id: None,
     };
     process_command(&config).unwrap();
-    check_recent_balance(49_989, &rpc_client, &sender_pubkey);
-    check_recent_balance(10, &rpc_client, &recipient_pubkey);
+    check_balance!(sol_to_lamports(4.0) - 1, &rpc_client, &sender_pubkey);
+    check_balance!(sol_to_lamports(1.0), &rpc_client, &recipient_pubkey);
 
     // Plain ole transfer, failure due to InsufficientFundsForSpendAndFee
     config.command = CliCommand::Transfer {
-        amount: SpendAmount::Some(49_989),
+        amount: SpendAmount::Some(sol_to_lamports(4.0)),
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
@@ -94,8 +98,8 @@ fn test_transfer() {
         derived_address_program_id: None,
     };
     assert!(process_command(&config).is_err());
-    check_recent_balance(49_989, &rpc_client, &sender_pubkey);
-    check_recent_balance(10, &rpc_client, &recipient_pubkey);
+    check_balance!(sol_to_lamports(4.0) - 1, &rpc_client, &sender_pubkey);
+    check_balance!(sol_to_lamports(1.0), &rpc_client, &recipient_pubkey);
 
     let mut offline = CliConfig::recent_for_tests();
     offline.json_rpc_url = String::default();
@@ -105,13 +109,14 @@ fn test_transfer() {
     process_command(&offline).unwrap_err();
 
     let offline_pubkey = offline.signers[0].pubkey();
-    request_and_confirm_airdrop(&rpc_client, &offline, &offline_pubkey, 50).unwrap();
-    check_recent_balance(50, &rpc_client, &offline_pubkey);
+    request_and_confirm_airdrop(&rpc_client, &offline, &offline_pubkey, sol_to_lamports(1.0))
+        .unwrap();
+    check_balance!(sol_to_lamports(1.0), &rpc_client, &offline_pubkey);
 
     // Offline transfer
     let blockhash = rpc_client.get_latest_blockhash().unwrap();
     offline.command = CliCommand::Transfer {
-        amount: SpendAmount::Some(10),
+        amount: SpendAmount::Some(sol_to_lamports(0.5)),
         to: recipient_pubkey,
         from: 0,
         sign_only: true,
@@ -133,7 +138,7 @@ fn test_transfer() {
     let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
     config.signers = vec![&offline_presigner];
     config.command = CliCommand::Transfer {
-        amount: SpendAmount::Some(10),
+        amount: SpendAmount::Some(sol_to_lamports(0.5)),
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
@@ -149,8 +154,8 @@ fn test_transfer() {
         derived_address_program_id: None,
     };
     process_command(&config).unwrap();
-    check_recent_balance(39, &rpc_client, &offline_pubkey);
-    check_recent_balance(20, &rpc_client, &recipient_pubkey);
+    check_balance!(sol_to_lamports(0.5) - 1, &rpc_client, &offline_pubkey);
+    check_balance!(sol_to_lamports(1.5), &rpc_client, &recipient_pubkey);
 
     // Create nonce account
     let nonce_account = keypair_from_seed(&[3u8; 32]).unwrap();
@@ -166,7 +171,11 @@ fn test_transfer() {
         amount: SpendAmount::Some(minimum_nonce_balance),
     };
     process_command(&config).unwrap();
-    check_recent_balance(49_987 - minimum_nonce_balance, &rpc_client, &sender_pubkey);
+    check_balance!(
+        sol_to_lamports(4.0) - 3 - minimum_nonce_balance,
+        &rpc_client,
+        &sender_pubkey,
+    );
 
     // Fetch nonce hash
     let nonce_hash = nonce_utils::get_account_with_commitment(
@@ -181,7 +190,7 @@ fn test_transfer() {
     // Nonced transfer
     config.signers = vec![&default_signer];
     config.command = CliCommand::Transfer {
-        amount: SpendAmount::Some(10),
+        amount: SpendAmount::Some(sol_to_lamports(1.0)),
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
@@ -200,8 +209,12 @@ fn test_transfer() {
         derived_address_program_id: None,
     };
     process_command(&config).unwrap();
-    check_recent_balance(49_976 - minimum_nonce_balance, &rpc_client, &sender_pubkey);
-    check_recent_balance(30, &rpc_client, &recipient_pubkey);
+    check_balance!(
+        sol_to_lamports(3.0) - 4 - minimum_nonce_balance,
+        &rpc_client,
+        &sender_pubkey,
+    );
+    check_balance!(sol_to_lamports(2.5), &rpc_client, &recipient_pubkey);
     let new_nonce_hash = nonce_utils::get_account_with_commitment(
         &rpc_client,
         &nonce_account.pubkey(),
@@ -221,7 +234,11 @@ fn test_transfer() {
         new_authority: offline_pubkey,
     };
     process_command(&config).unwrap();
-    check_recent_balance(49_975 - minimum_nonce_balance, &rpc_client, &sender_pubkey);
+    check_balance!(
+        sol_to_lamports(3.0) - 5 - minimum_nonce_balance,
+        &rpc_client,
+        &sender_pubkey,
+    );
 
     // Fetch nonce hash
     let nonce_hash = nonce_utils::get_account_with_commitment(
@@ -236,7 +253,7 @@ fn test_transfer() {
     // Offline, nonced transfer
     offline.signers = vec![&default_offline_signer];
     offline.command = CliCommand::Transfer {
-        amount: SpendAmount::Some(10),
+        amount: SpendAmount::Some(sol_to_lamports(0.4)),
         to: recipient_pubkey,
         from: 0,
         sign_only: true,
@@ -257,7 +274,7 @@ fn test_transfer() {
     let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
     config.signers = vec![&offline_presigner];
     config.command = CliCommand::Transfer {
-        amount: SpendAmount::Some(10),
+        amount: SpendAmount::Some(sol_to_lamports(0.4)),
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
@@ -276,8 +293,8 @@ fn test_transfer() {
         derived_address_program_id: None,
     };
     process_command(&config).unwrap();
-    check_recent_balance(28, &rpc_client, &offline_pubkey);
-    check_recent_balance(40, &rpc_client, &recipient_pubkey);
+    check_balance!(sol_to_lamports(0.1) - 2, &rpc_client, &offline_pubkey);
+    check_balance!(sol_to_lamports(2.9), &rpc_client, &recipient_pubkey);
 }
 
 #[test]
@@ -305,19 +322,27 @@ fn test_transfer_multisession_signing() {
         &rpc_client,
         &CliConfig::recent_for_tests(),
         &offline_from_signer.pubkey(),
-        43,
+        sol_to_lamports(43.0),
     )
     .unwrap();
     request_and_confirm_airdrop(
         &rpc_client,
         &CliConfig::recent_for_tests(),
         &offline_fee_payer_signer.pubkey(),
-        3,
+        sol_to_lamports(1.0) + 3,
     )
     .unwrap();
-    check_recent_balance(43, &rpc_client, &offline_from_signer.pubkey());
-    check_recent_balance(3, &rpc_client, &offline_fee_payer_signer.pubkey());
-    check_recent_balance(0, &rpc_client, &to_pubkey);
+    check_balance!(
+        sol_to_lamports(43.0),
+        &rpc_client,
+        &offline_from_signer.pubkey(),
+    );
+    check_balance!(
+        sol_to_lamports(1.0) + 3,
+        &rpc_client,
+        &offline_fee_payer_signer.pubkey(),
+    );
+    check_balance!(0, &rpc_client, &to_pubkey);
 
     check_ready(&rpc_client);
 
@@ -331,7 +356,7 @@ fn test_transfer_multisession_signing() {
     fee_payer_config.command = CliCommand::ClusterVersion;
     process_command(&fee_payer_config).unwrap_err();
     fee_payer_config.command = CliCommand::Transfer {
-        amount: SpendAmount::Some(42),
+        amount: SpendAmount::Some(sol_to_lamports(42.0)),
         to: to_pubkey,
         from: 1,
         sign_only: true,
@@ -362,7 +387,7 @@ fn test_transfer_multisession_signing() {
     from_config.command = CliCommand::ClusterVersion;
     process_command(&from_config).unwrap_err();
     from_config.command = CliCommand::Transfer {
-        amount: SpendAmount::Some(42),
+        amount: SpendAmount::Some(sol_to_lamports(42.0)),
         to: to_pubkey,
         from: 1,
         sign_only: true,
@@ -390,7 +415,7 @@ fn test_transfer_multisession_signing() {
     config.json_rpc_url = test_validator.rpc_url();
     config.signers = vec![&fee_payer_presigner, &from_presigner];
     config.command = CliCommand::Transfer {
-        amount: SpendAmount::Some(42),
+        amount: SpendAmount::Some(sol_to_lamports(42.0)),
         to: to_pubkey,
         from: 1,
         sign_only: false,
@@ -407,9 +432,17 @@ fn test_transfer_multisession_signing() {
     };
     process_command(&config).unwrap();
 
-    check_recent_balance(1, &rpc_client, &offline_from_signer.pubkey());
-    check_recent_balance(1, &rpc_client, &offline_fee_payer_signer.pubkey());
-    check_recent_balance(42, &rpc_client, &to_pubkey);
+    check_balance!(
+        sol_to_lamports(1.0),
+        &rpc_client,
+        &offline_from_signer.pubkey(),
+    );
+    check_balance!(
+        sol_to_lamports(1.0) + 1,
+        &rpc_client,
+        &offline_fee_payer_signer.pubkey(),
+    );
+    check_balance!(sol_to_lamports(42.0), &rpc_client, &to_pubkey);
 }
 
 #[test]
@@ -438,8 +471,8 @@ fn test_transfer_all() {
     let recipient_pubkey = Pubkey::new(&[1u8; 32]);
 
     request_and_confirm_airdrop(&rpc_client, &config, &sender_pubkey, 50_000).unwrap();
-    check_recent_balance(50_000, &rpc_client, &sender_pubkey);
-    check_recent_balance(0, &rpc_client, &recipient_pubkey);
+    check_balance!(50_000, &rpc_client, &sender_pubkey);
+    check_balance!(0, &rpc_client, &recipient_pubkey);
 
     check_ready(&rpc_client);
 
@@ -461,8 +494,8 @@ fn test_transfer_all() {
         derived_address_program_id: None,
     };
     process_command(&config).unwrap();
-    check_recent_balance(0, &rpc_client, &sender_pubkey);
-    check_recent_balance(49_999, &rpc_client, &recipient_pubkey);
+    check_balance!(0, &rpc_client, &sender_pubkey);
+    check_balance!(49_999, &rpc_client, &recipient_pubkey);
 }
 
 #[test]
@@ -491,8 +524,8 @@ fn test_transfer_unfunded_recipient() {
     let recipient_pubkey = Pubkey::new(&[1u8; 32]);
 
     request_and_confirm_airdrop(&rpc_client, &config, &sender_pubkey, 50_000).unwrap();
-    check_recent_balance(50_000, &rpc_client, &sender_pubkey);
-    check_recent_balance(0, &rpc_client, &recipient_pubkey);
+    check_balance!(50_000, &rpc_client, &sender_pubkey);
+    check_balance!(0, &rpc_client, &recipient_pubkey);
 
     check_ready(&rpc_client);
 
@@ -551,17 +584,19 @@ fn test_transfer_with_seed() {
     )
     .unwrap();
 
-    request_and_confirm_airdrop(&rpc_client, &config, &sender_pubkey, 1).unwrap();
-    request_and_confirm_airdrop(&rpc_client, &config, &derived_address, 50_000).unwrap();
-    check_recent_balance(1, &rpc_client, &sender_pubkey);
-    check_recent_balance(50_000, &rpc_client, &derived_address);
-    check_recent_balance(0, &rpc_client, &recipient_pubkey);
+    request_and_confirm_airdrop(&rpc_client, &config, &sender_pubkey, sol_to_lamports(1.0))
+        .unwrap();
+    request_and_confirm_airdrop(&rpc_client, &config, &derived_address, sol_to_lamports(5.0))
+        .unwrap();
+    check_balance!(sol_to_lamports(1.0), &rpc_client, &sender_pubkey);
+    check_balance!(sol_to_lamports(5.0), &rpc_client, &derived_address);
+    check_balance!(0, &rpc_client, &recipient_pubkey);
 
     check_ready(&rpc_client);
 
     // Transfer with seed
     config.command = CliCommand::Transfer {
-        amount: SpendAmount::Some(50_000),
+        amount: SpendAmount::Some(sol_to_lamports(5.0)),
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
@@ -577,7 +612,7 @@ fn test_transfer_with_seed() {
         derived_address_program_id: Some(derived_address_program_id),
     };
     process_command(&config).unwrap();
-    check_recent_balance(0, &rpc_client, &sender_pubkey);
-    check_recent_balance(50_000, &rpc_client, &recipient_pubkey);
-    check_recent_balance(0, &rpc_client, &derived_address);
+    check_balance!(sol_to_lamports(1.0) - 1, &rpc_client, &sender_pubkey);
+    check_balance!(sol_to_lamports(5.0), &rpc_client, &recipient_pubkey);
+    check_balance!(0, &rpc_client, &derived_address);
 }

cli/tests/vote.rs

@@ -1,9 +1,11 @@
+#![allow(clippy::integer_arithmetic)]
 use {
     solana_cli::{
+        check_balance,
         cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig},
         spend_utils::SpendAmount,
-        test_utils::check_recent_balance,
     },
+    solana_cli_output::{parse_sign_only_reply_string, OutputFormat},
     solana_client::{
         blockhash_query::{self, BlockhashQuery},
         rpc_client::RpcClient,
@@ -12,7 +14,7 @@ use {
     solana_sdk::{
         account_utils::StateMut,
         commitment_config::CommitmentConfig,
-        signature::{Keypair, Signer},
+        signature::{Keypair, NullSigner, Signer},
     },
     solana_streamer::socket::SocketAddrSpace,
     solana_test_validator::TestValidator,
@@ -49,7 +51,13 @@ fn test_vote_authorize_and_withdraw() {
         authorized_voter: None,
         authorized_withdrawer: config.signers[0].pubkey(),
         commission: 0,
+        sign_only: false,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+        nonce_account: None,
+        nonce_authority: 0,
         memo: None,
+        fee_payer: 0,
     };
     process_command(&config).unwrap();
     let vote_account = rpc_client
@@ -62,12 +70,12 @@ fn test_vote_authorize_and_withdraw() {
         .get_minimum_balance_for_rent_exemption(VoteState::size_of())
         .unwrap()
         .max(1);
-    check_recent_balance(expected_balance, &rpc_client, &vote_account_pubkey);
+    check_balance!(expected_balance, &rpc_client, &vote_account_pubkey);
 
     // Transfer in some more SOL
     config.signers = vec![&default_signer];
     config.command = CliCommand::Transfer {
-        amount: SpendAmount::Some(1_000),
+        amount: SpendAmount::Some(10_000),
         to: vote_account_pubkey,
         from: 0,
         sign_only: false,
@@ -83,8 +91,8 @@ fn test_vote_authorize_and_withdraw() {
         derived_address_program_id: None,
     };
     process_command(&config).unwrap();
-    let expected_balance = expected_balance + 1_000;
-    check_recent_balance(expected_balance, &rpc_client, &vote_account_pubkey);
+    let expected_balance = expected_balance + 10_000;
+    check_balance!(expected_balance, &rpc_client, &vote_account_pubkey);
 
     // Authorize vote account withdrawal to another signer
     let first_withdraw_authority = Keypair::new();
@@ -93,7 +101,13 @@ fn test_vote_authorize_and_withdraw() {
         vote_account_pubkey,
         new_authorized_pubkey: first_withdraw_authority.pubkey(),
         vote_authorize: VoteAuthorize::Withdrawer,
+        sign_only: false,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+        nonce_account: None,
+        nonce_authority: 0,
         memo: None,
+        fee_payer: 0,
         authorized: 0,
         new_authorized: None,
     };
@@ -112,7 +126,13 @@ fn test_vote_authorize_and_withdraw() {
         vote_account_pubkey,
         new_authorized_pubkey: withdraw_authority.pubkey(),
         vote_authorize: VoteAuthorize::Withdrawer,
+        sign_only: false,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+        nonce_account: None,
+        nonce_authority: 0,
         memo: None,
+        fee_payer: 0,
         authorized: 1,
         new_authorized: Some(1),
     };
@@ -126,7 +146,13 @@ fn test_vote_authorize_and_withdraw() {
         vote_account_pubkey,
         new_authorized_pubkey: withdraw_authority.pubkey(),
         vote_authorize: VoteAuthorize::Withdrawer,
+        sign_only: false,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+        nonce_account: None,
+        nonce_authority: 0,
         memo: None,
+        fee_payer: 0,
         authorized: 1,
         new_authorized: Some(2),
     };
@@ -144,14 +170,20 @@ fn test_vote_authorize_and_withdraw() {
     config.command = CliCommand::WithdrawFromVoteAccount {
         vote_account_pubkey,
         withdraw_authority: 1,
-        withdraw_amount: SpendAmount::Some(100),
+        withdraw_amount: SpendAmount::Some(1_000),
         destination_account_pubkey: destination_account,
+        sign_only: false,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+        nonce_account: None,
+        nonce_authority: 0,
         memo: None,
+        fee_payer: 0,
     };
     process_command(&config).unwrap();
-    let expected_balance = expected_balance - 100;
-    check_recent_balance(expected_balance, &rpc_client, &vote_account_pubkey);
-    check_recent_balance(100, &rpc_client, &destination_account);
+    let expected_balance = expected_balance - 1_000;
+    check_balance!(expected_balance, &rpc_client, &vote_account_pubkey);
+    check_balance!(1_000, &rpc_client, &destination_account);
 
     // Re-assign validator identity
     let new_identity_keypair = Keypair::new();
@@ -160,7 +192,13 @@ fn test_vote_authorize_and_withdraw() {
         vote_account_pubkey,
         new_identity_account: 2,
         withdraw_authority: 1,
+        sign_only: false,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+        nonce_account: None,
+        nonce_authority: 0,
         memo: None,
+        fee_payer: 0,
     };
     process_command(&config).unwrap();
 
@@ -172,8 +210,281 @@ fn test_vote_authorize_and_withdraw() {
         withdraw_authority: 1,
         destination_account_pubkey: destination_account,
         memo: None,
+        fee_payer: 0,
     };
     process_command(&config).unwrap();
-    check_recent_balance(0, &rpc_client, &vote_account_pubkey);
-    check_recent_balance(expected_balance, &rpc_client, &destination_account);
+    check_balance!(0, &rpc_client, &vote_account_pubkey);
+    check_balance!(expected_balance, &rpc_client, &destination_account);
+}
+
+#[test]
+fn test_offline_vote_authorize_and_withdraw() {
+    let mint_keypair = Keypair::new();
+    let mint_pubkey = mint_keypair.pubkey();
+    let faucet_addr = run_local_faucet(mint_keypair, None);
+    let test_validator =
+        TestValidator::with_no_fees(mint_pubkey, Some(faucet_addr), SocketAddrSpace::Unspecified);
+
+    let rpc_client =
+        RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
+    let default_signer = Keypair::new();
+
+    let mut config_payer = CliConfig::recent_for_tests();
+    config_payer.json_rpc_url = test_validator.rpc_url();
+    config_payer.signers = vec![&default_signer];
+
+    let mut config_offline = CliConfig::recent_for_tests();
+    config_offline.json_rpc_url = String::default();
+    config_offline.command = CliCommand::ClusterVersion;
+    let offline_keypair = Keypair::new();
+    config_offline.signers = vec![&offline_keypair];
+    // Verify that we cannot reach the cluster
+    process_command(&config_offline).unwrap_err();
+
+    request_and_confirm_airdrop(
+        &rpc_client,
+        &config_payer,
+        &config_payer.signers[0].pubkey(),
+        100_000,
+    )
+    .unwrap();
+    check_balance!(100_000, &rpc_client, &config_payer.signers[0].pubkey());
+
+    request_and_confirm_airdrop(
+        &rpc_client,
+        &config_offline,
+        &config_offline.signers[0].pubkey(),
+        100_000,
+    )
+    .unwrap();
+    check_balance!(100_000, &rpc_client, &config_offline.signers[0].pubkey());
+
+    // Create vote account with specific withdrawer
+    let vote_account_keypair = Keypair::new();
+    let vote_account_pubkey = vote_account_keypair.pubkey();
+    config_payer.signers = vec![&default_signer, &vote_account_keypair];
+    config_payer.command = CliCommand::CreateVoteAccount {
+        vote_account: 1,
+        seed: None,
+        identity_account: 0,
+        authorized_voter: None,
+        authorized_withdrawer: offline_keypair.pubkey(),
+        commission: 0,
+        sign_only: false,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+        nonce_account: None,
+        nonce_authority: 0,
+        memo: None,
+        fee_payer: 0,
+    };
+    process_command(&config_payer).unwrap();
+    let vote_account = rpc_client
+        .get_account(&vote_account_keypair.pubkey())
+        .unwrap();
+    let vote_state: VoteStateVersions = vote_account.state().unwrap();
+    let authorized_withdrawer = vote_state.convert_to_current().authorized_withdrawer;
+    assert_eq!(authorized_withdrawer, offline_keypair.pubkey());
+    let expected_balance = rpc_client
+        .get_minimum_balance_for_rent_exemption(VoteState::size_of())
+        .unwrap()
+        .max(1);
+    check_balance!(expected_balance, &rpc_client, &vote_account_pubkey);
+
+    // Transfer in some more SOL
+    config_payer.signers = vec![&default_signer];
+    config_payer.command = CliCommand::Transfer {
+        amount: SpendAmount::Some(10_000),
+        to: vote_account_pubkey,
+        from: 0,
+        sign_only: false,
+        dump_transaction_message: false,
+        allow_unfunded_recipient: true,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+        nonce_account: None,
+        nonce_authority: 0,
+        memo: None,
+        fee_payer: 0,
+        derived_address_seed: None,
+        derived_address_program_id: None,
+    };
+    process_command(&config_payer).unwrap();
+    let expected_balance = expected_balance + 10_000;
+    check_balance!(expected_balance, &rpc_client, &vote_account_pubkey);
+
+    // Authorize vote account withdrawal to another signer, offline
+    let withdraw_authority = Keypair::new();
+    let blockhash = rpc_client.get_latest_blockhash().unwrap();
+    config_offline.command = CliCommand::VoteAuthorize {
+        vote_account_pubkey,
+        new_authorized_pubkey: withdraw_authority.pubkey(),
+        vote_authorize: VoteAuthorize::Withdrawer,
+        sign_only: true,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::None(blockhash),
+        nonce_account: None,
+        nonce_authority: 0,
+        memo: None,
+        fee_payer: 0,
+        authorized: 0,
+        new_authorized: None,
+    };
+    config_offline.output_format = OutputFormat::JsonCompact;
+    let sig_response = process_command(&config_offline).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only
+        .presigner_of(&config_offline.signers[0].pubkey())
+        .unwrap();
+    config_payer.signers = vec![&offline_presigner];
+    config_payer.command = CliCommand::VoteAuthorize {
+        vote_account_pubkey,
+        new_authorized_pubkey: withdraw_authority.pubkey(),
+        vote_authorize: VoteAuthorize::Withdrawer,
+        sign_only: false,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
+        nonce_account: None,
+        nonce_authority: 0,
+        memo: None,
+        fee_payer: 0,
+        authorized: 0,
+        new_authorized: None,
+    };
+    process_command(&config_payer).unwrap();
+    let vote_account = rpc_client
+        .get_account(&vote_account_keypair.pubkey())
+        .unwrap();
+    let vote_state: VoteStateVersions = vote_account.state().unwrap();
+    let authorized_withdrawer = vote_state.convert_to_current().authorized_withdrawer;
+    assert_eq!(authorized_withdrawer, withdraw_authority.pubkey());
+
+    // Withdraw from vote account offline
+    let destination_account = solana_sdk::pubkey::new_rand(); // Send withdrawal to new account to make balance check easy
+    let blockhash = rpc_client.get_latest_blockhash().unwrap();
+    let fee_payer_null_signer = NullSigner::new(&default_signer.pubkey());
+    config_offline.signers = vec![&fee_payer_null_signer, &withdraw_authority];
+    config_offline.command = CliCommand::WithdrawFromVoteAccount {
+        vote_account_pubkey,
+        withdraw_authority: 1,
+        withdraw_amount: SpendAmount::Some(1_000),
+        destination_account_pubkey: destination_account,
+        sign_only: true,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::None(blockhash),
+        nonce_account: None,
+        nonce_authority: 0,
+        memo: None,
+        fee_payer: 0,
+    };
+    config_offline.output_format = OutputFormat::JsonCompact;
+    let sig_response = process_command(&config_offline).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    let offline_presigner = sign_only
+        .presigner_of(&config_offline.signers[1].pubkey())
+        .unwrap();
+    config_payer.signers = vec![&default_signer, &offline_presigner];
+    config_payer.command = CliCommand::WithdrawFromVoteAccount {
+        vote_account_pubkey,
+        withdraw_authority: 1,
+        withdraw_amount: SpendAmount::Some(1_000),
+        destination_account_pubkey: destination_account,
+        sign_only: false,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
+        nonce_account: None,
+        nonce_authority: 0,
+        memo: None,
+        fee_payer: 0,
+    };
+    process_command(&config_payer).unwrap();
+    let expected_balance = expected_balance - 1_000;
+    check_balance!(expected_balance, &rpc_client, &vote_account_pubkey);
+    check_balance!(1_000, &rpc_client, &destination_account);
+
+    // Re-assign validator identity offline
+    let blockhash = rpc_client.get_latest_blockhash().unwrap();
+    let new_identity_keypair = Keypair::new();
+    let new_identity_null_signer = NullSigner::new(&new_identity_keypair.pubkey());
+    config_offline.signers = vec![
+        &fee_payer_null_signer,
+        &withdraw_authority,
+        &new_identity_null_signer,
+    ];
+    config_offline.command = CliCommand::VoteUpdateValidator {
+        vote_account_pubkey,
+        new_identity_account: 2,
+        withdraw_authority: 1,
+        sign_only: true,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::None(blockhash),
+        nonce_account: None,
+        nonce_authority: 0,
+        memo: None,
+        fee_payer: 0,
+    };
+    process_command(&config_offline).unwrap();
+    config_offline.output_format = OutputFormat::JsonCompact;
+    let sig_response = process_command(&config_offline).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    let offline_presigner = sign_only
+        .presigner_of(&config_offline.signers[1].pubkey())
+        .unwrap();
+    config_payer.signers = vec![&default_signer, &offline_presigner, &new_identity_keypair];
+    config_payer.command = CliCommand::VoteUpdateValidator {
+        vote_account_pubkey,
+        new_identity_account: 2,
+        withdraw_authority: 1,
+        sign_only: false,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
+        nonce_account: None,
+        nonce_authority: 0,
+        memo: None,
+        fee_payer: 0,
+    };
+    process_command(&config_payer).unwrap();
+
+    // Close vote account offline. Must use WithdrawFromVoteAccount and specify amount, since
+    // CloseVoteAccount requires RpcClient
+    let destination_account = solana_sdk::pubkey::new_rand(); // Send withdrawal to new account to make balance check easy
+    config_offline.signers = vec![&fee_payer_null_signer, &withdraw_authority];
+    config_offline.command = CliCommand::WithdrawFromVoteAccount {
+        vote_account_pubkey,
+        withdraw_authority: 1,
+        withdraw_amount: SpendAmount::Some(expected_balance),
+        destination_account_pubkey: destination_account,
+        sign_only: true,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::None(blockhash),
+        nonce_account: None,
+        nonce_authority: 0,
+        memo: None,
+        fee_payer: 0,
+    };
+    process_command(&config_offline).unwrap();
+    config_offline.output_format = OutputFormat::JsonCompact;
+    let sig_response = process_command(&config_offline).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    let offline_presigner = sign_only
+        .presigner_of(&config_offline.signers[1].pubkey())
+        .unwrap();
+    config_payer.signers = vec![&default_signer, &offline_presigner];
+    config_payer.command = CliCommand::WithdrawFromVoteAccount {
+        vote_account_pubkey,
+        withdraw_authority: 1,
+        withdraw_amount: SpendAmount::Some(expected_balance),
+        destination_account_pubkey: destination_account,
+        sign_only: false,
+        dump_transaction_message: false,
+        blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
+        nonce_account: None,
+        nonce_authority: 0,
+        memo: None,
+        fee_payer: 0,
+    };
+    process_command(&config_payer).unwrap();
+    check_balance!(0, &rpc_client, &vote_account_pubkey);
+    check_balance!(expected_balance, &rpc_client, &destination_account);
 }

client-test/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client-test"
-version = "1.9.0"
+version = "1.10.0"
 description = "Solana RPC Test"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,24 +10,26 @@ documentation = "https://docs.rs/solana-client-test"
 edition = "2021"
 
 [dependencies]
-serde_json = "1.0.72"
+serde_json = "1.0.74"
 serial_test = "0.5.1"
-solana-client = { path = "../client", version = "=1.9.0" }
-solana-measure = { path = "../measure", version = "=1.9.0" }
-solana-merkle-tree = { path = "../merkle-tree", version = "=1.9.0" }
-solana-metrics = { path = "../metrics", version = "=1.9.0" }
-solana-perf = { path = "../perf", version = "=1.9.0" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.9.0" }
-solana-rpc = { path = "../rpc", version = "=1.9.0" }
-solana-runtime = { path = "../runtime", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-solana-streamer = { path = "../streamer", version = "=1.9.0" }
-solana-test-validator = { path = "../test-validator", version = "=1.9.0" }
-solana-version = { path = "../version", version = "=1.9.0" }
+solana-client = { path = "../client", version = "=1.10.0" }
+solana-ledger = { path = "../ledger", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-merkle-tree = { path = "../merkle-tree", version = "=1.10.0" }
+solana-metrics = { path = "../metrics", version = "=1.10.0" }
+solana-perf = { path = "../perf", version = "=1.10.0" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.10.0" }
+solana-rpc = { path = "../rpc", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-streamer = { path = "../streamer", version = "=1.10.0" }
+solana-test-validator = { path = "../test-validator", version = "=1.10.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
+solana-version = { path = "../version", version = "=1.10.0" }
 systemstat = "0.1.10"
 
 [dev-dependencies]
-solana-logger = { path = "../logger", version = "=1.9.0" }
+solana-logger = { path = "../logger", version = "=1.10.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client-test/tests/client.rs

@@ -4,11 +4,16 @@ use {
     solana_client::{
         pubsub_client::PubsubClient,
         rpc_client::RpcClient,
-        rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig},
-        rpc_response::SlotInfo,
+        rpc_config::{
+            RpcAccountInfoConfig, RpcBlockSubscribeConfig, RpcBlockSubscribeFilter,
+            RpcProgramAccountsConfig,
+        },
+        rpc_response::{RpcBlockUpdate, SlotInfo},
     },
+    solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path},
     solana_rpc::{
         optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank,
+        rpc::create_test_transactions_and_populate_blockstore,
         rpc_pubsub_service::{PubSubConfig, PubSubService},
         rpc_subscriptions::RpcSubscriptions,
     },
@@ -20,7 +25,7 @@ use {
     },
     solana_sdk::{
         clock::Slot,
-        commitment_config::CommitmentConfig,
+        commitment_config::{CommitmentConfig, CommitmentLevel},
         native_token::sol_to_lamports,
         pubkey::Pubkey,
         rpc_port,
@@ -29,11 +34,12 @@ use {
     },
     solana_streamer::socket::SocketAddrSpace,
     solana_test_validator::TestValidator,
+    solana_transaction_status::{TransactionDetails, UiTransactionEncoding},
     std::{
         collections::HashSet,
         net::{IpAddr, SocketAddr},
         sync::{
-            atomic::{AtomicBool, Ordering},
+            atomic::{AtomicBool, AtomicU64, Ordering},
             Arc, RwLock,
         },
         thread::sleep,
@@ -119,9 +125,10 @@ fn test_account_subscription() {
     let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
     bank_forks.write().unwrap().insert(bank1);
     let bob = Keypair::new();
-
+    let max_complete_transaction_status_slot = Arc::new(AtomicU64::default());
     let subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
         &exit,
+        max_complete_transaction_status_slot,
         bank_forks.clone(),
         Arc::new(RwLock::new(BlockCommitmentCache::default())),
         OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
@@ -194,6 +201,113 @@ fn test_account_subscription() {
     assert_eq!(errors, [].to_vec());
 }
 
+#[test]
+#[serial]
+fn test_block_subscription() {
+    // setup BankForks
+    let exit = Arc::new(AtomicBool::new(false));
+    let GenesisConfigInfo {
+        genesis_config,
+        mint_keypair: alice,
+        ..
+    } = create_genesis_config(10_000);
+    let bank = Bank::new_for_tests(&genesis_config);
+    let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
+
+    // setup Blockstore
+    let ledger_path = get_tmp_ledger_path!();
+    let blockstore = Blockstore::open(&ledger_path).unwrap();
+    let blockstore = Arc::new(blockstore);
+
+    // populate ledger with test txs
+    let bank = bank_forks.read().unwrap().working_bank();
+    let keypair1 = Keypair::new();
+    let keypair2 = Keypair::new();
+    let keypair3 = Keypair::new();
+    let max_complete_transaction_status_slot = Arc::new(AtomicU64::new(blockstore.max_root()));
+    let _confirmed_block_signatures = create_test_transactions_and_populate_blockstore(
+        vec![&alice, &keypair1, &keypair2, &keypair3],
+        0,
+        bank,
+        blockstore.clone(),
+        max_complete_transaction_status_slot,
+    );
+    let max_complete_transaction_status_slot = Arc::new(AtomicU64::default());
+    // setup RpcSubscriptions && PubSubService
+    let subscriptions = Arc::new(RpcSubscriptions::new_for_tests_with_blockstore(
+        &exit,
+        max_complete_transaction_status_slot,
+        blockstore.clone(),
+        bank_forks.clone(),
+        Arc::new(RwLock::new(BlockCommitmentCache::default())),
+        OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
+    ));
+    let pubsub_addr = SocketAddr::new(
+        IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
+        rpc_port::DEFAULT_RPC_PUBSUB_PORT,
+    );
+    let pub_cfg = PubSubConfig {
+        enable_block_subscription: true,
+        ..PubSubConfig::default()
+    };
+    let (trigger, pubsub_service) = PubSubService::new(pub_cfg, &subscriptions, pubsub_addr);
+
+    std::thread::sleep(Duration::from_millis(400));
+
+    // setup PubsubClient
+    let (mut client, receiver) = PubsubClient::block_subscribe(
+        &format!("ws://0.0.0.0:{}/", pubsub_addr.port()),
+        RpcBlockSubscribeFilter::All,
+        Some(RpcBlockSubscribeConfig {
+            commitment: Some(CommitmentConfig {
+                commitment: CommitmentLevel::Confirmed,
+            }),
+            encoding: Some(UiTransactionEncoding::Json),
+            transaction_details: Some(TransactionDetails::Signatures),
+            show_rewards: None,
+        }),
+    )
+    .unwrap();
+
+    // trigger Gossip notification
+    let slot = bank_forks.read().unwrap().highest_slot();
+    subscriptions.notify_gossip_subscribers(slot);
+    let maybe_actual = receiver.recv_timeout(Duration::from_millis(400));
+    match maybe_actual {
+        Ok(actual) => {
+            let versioned_block = blockstore.get_complete_block(slot, false).unwrap();
+            let legacy_block = versioned_block.into_legacy_block().unwrap();
+            let block = legacy_block.clone().configure(
+                UiTransactionEncoding::Json,
+                TransactionDetails::Signatures,
+                false,
+            );
+            let expected = RpcBlockUpdate {
+                slot,
+                block: Some(block),
+                err: None,
+            };
+            let block = legacy_block.configure(
+                UiTransactionEncoding::Json,
+                TransactionDetails::Signatures,
+                false,
+            );
+            assert_eq!(actual.value.slot, expected.slot);
+            assert!(block.eq(&actual.value.block.unwrap()));
+        }
+        Err(e) => {
+            eprintln!("unexpected websocket receive timeout");
+            assert_eq!(Some(e), None);
+        }
+    }
+
+    // cleanup
+    exit.store(true, Ordering::Relaxed);
+    trigger.cancel();
+    client.shutdown().unwrap();
+    pubsub_service.close().unwrap();
+}
+
 #[test]
 #[serial]
 fn test_program_subscription() {
@@ -215,9 +329,10 @@ fn test_program_subscription() {
     let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
     bank_forks.write().unwrap().insert(bank1);
     let bob = Keypair::new();
-
+    let max_complete_transaction_status_slot = Arc::new(AtomicU64::default());
     let subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
         &exit,
+        max_complete_transaction_status_slot,
         bank_forks.clone(),
         Arc::new(RwLock::new(BlockCommitmentCache::default())),
         OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
@@ -300,9 +415,10 @@ fn test_root_subscription() {
     let bank0 = bank_forks.read().unwrap().get(0).unwrap().clone();
     let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
     bank_forks.write().unwrap().insert(bank1);
-
+    let max_complete_transaction_status_slot = Arc::new(AtomicU64::default());
     let subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
         &exit,
+        max_complete_transaction_status_slot,
         bank_forks.clone(),
         Arc::new(RwLock::new(BlockCommitmentCache::default())),
         OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
@@ -350,8 +466,10 @@ fn test_slot_subscription() {
     let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
     let optimistically_confirmed_bank =
         OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks);
+    let max_complete_transaction_status_slot = Arc::new(AtomicU64::default());
     let subscriptions = Arc::new(RpcSubscriptions::new_for_tests(
         &exit,
+        max_complete_transaction_status_slot,
         bank_forks,
         Arc::new(RwLock::new(BlockCommitmentCache::default())),
         optimistically_confirmed_bank,

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.9.0"
+version = "1.10.0"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,24 +14,25 @@ base64 = "0.13.0"
 bincode = "1.3.3"
 bs58 = "0.4.0"
 clap = "2.33.0"
+crossbeam-channel = "0.5"
 indicatif = "0.16.2"
 jsonrpc-core = "18.0.0"
 log = "0.4.14"
 rayon = "1.5.1"
 reqwest = { version = "0.11.6", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 semver = "1.0.4"
-serde = "1.0.130"
+serde = "1.0.133"
 serde_derive = "1.0.103"
-serde_json = "1.0.72"
-solana-account-decoder = { path = "../account-decoder", version = "=1.9.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.9.0" }
-solana-faucet = { path = "../faucet", version = "=1.9.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.9.0" }
-solana-measure = { path = "../measure", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.9.0" }
-solana-version = { path = "../version", version = "=1.9.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.9.0" }
+serde_json = "1.0.74"
+solana-account-decoder = { path = "../account-decoder", version = "=1.10.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.10.0" }
+solana-faucet = { path = "../faucet", version = "=1.10.0" }
+solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
+solana-version = { path = "../version", version = "=1.10.0" }
+solana-vote-program = { path = "../programs/vote", version = "=1.10.0" }
 thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 tungstenite = { version = "0.16.0", features = ["rustls-tls-webpki-roots"] }
@@ -40,7 +41,7 @@ url = "2.2.2"
 [dev-dependencies]
 assert_matches = "1.5.0"
 jsonrpc-http-server = "18.0.0"
-solana-logger = { path = "../logger", version = "=1.9.0" }
+solana-logger = { path = "../logger", version = "=1.10.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/mock_sender.rs

@@ -30,7 +30,7 @@ use {
         transaction::{self, Transaction, TransactionError},
     },
     solana_transaction_status::{
-        EncodedConfirmedBlock, EncodedConfirmedTransaction, EncodedTransaction,
+        EncodedConfirmedBlock, EncodedConfirmedTransactionWithStatusMeta, EncodedTransaction,
         EncodedTransactionWithStatusMeta, Rewards, TransactionConfirmationStatus,
         TransactionStatus, UiCompiledInstruction, UiMessage, UiRawMessage, UiTransaction,
         UiTransactionEncoding, UiTransactionStatusMeta,
@@ -185,7 +185,7 @@ impl RpcSender for MockSender {
                     value: statuses,
                 })?
             }
-            "getTransaction" => serde_json::to_value(EncodedConfirmedTransaction {
+            "getTransaction" => serde_json::to_value(EncodedConfirmedTransactionWithStatusMeta {
                 slot: 2,
                 transaction: EncodedTransactionWithStatusMeta {
                     transaction: EncodedTransaction::Json(

client/src/nonce_utils.rs

@@ -4,6 +4,7 @@ use {
         account::{Account, ReadableAccount},
         account_utils::StateMut,
         commitment_config::CommitmentConfig,
+        hash::Hash,
         nonce::{
             state::{Data, Versions},
             State,
@@ -21,10 +22,10 @@ pub enum Error {
     InvalidAccountData,
     #[error("unexpected account data size")]
     UnexpectedDataSize,
-    #[error("query hash does not match stored hash")]
-    InvalidHash,
-    #[error("query authority does not match account authority")]
-    InvalidAuthority,
+    #[error("provided hash ({provided}) does not match nonce hash ({expected})")]
+    InvalidHash { provided: Hash, expected: Hash },
+    #[error("provided authority ({provided}) does not match nonce authority ({expected})")]
+    InvalidAuthority { provided: Pubkey, expected: Pubkey },
     #[error("invalid state for requested operation")]
     InvalidStateForOperation,
     #[error("client error: {0}")]

client/src/pubsub_client.rs

@@ -1,14 +1,16 @@
 use {
     crate::{
         rpc_config::{
-            RpcAccountInfoConfig, RpcProgramAccountsConfig, RpcSignatureSubscribeConfig,
-            RpcTransactionLogsConfig, RpcTransactionLogsFilter,
+            RpcAccountInfoConfig, RpcBlockSubscribeConfig, RpcBlockSubscribeFilter,
+            RpcProgramAccountsConfig, RpcSignatureSubscribeConfig, RpcTransactionLogsConfig,
+            RpcTransactionLogsFilter,
         },
         rpc_response::{
-            Response as RpcResponse, RpcKeyedAccount, RpcLogsResponse, RpcSignatureResult,
-            SlotInfo, SlotUpdate,
+            Response as RpcResponse, RpcBlockUpdate, RpcKeyedAccount, RpcLogsResponse,
+            RpcSignatureResult, RpcVote, SlotInfo, SlotUpdate,
         },
     },
+    crossbeam_channel::{unbounded, Receiver, Sender},
     log::*,
     serde::de::DeserializeOwned,
     serde_json::{
@@ -23,7 +25,6 @@ use {
         net::TcpStream,
         sync::{
             atomic::{AtomicBool, Ordering},
-            mpsc::{channel, Receiver, Sender},
             Arc, RwLock,
         },
         thread::{sleep, JoinHandle},
@@ -173,6 +174,12 @@ pub type SignatureSubscription = (
     Receiver<RpcResponse<RpcSignatureResult>>,
 );
 
+pub type PubsubBlockClientSubscription = PubsubClientSubscription<RpcResponse<RpcBlockUpdate>>;
+pub type BlockSubscription = (
+    PubsubBlockClientSubscription,
+    Receiver<RpcResponse<RpcBlockUpdate>>,
+);
+
 pub type PubsubProgramClientSubscription = PubsubClientSubscription<RpcResponse<RpcKeyedAccount>>;
 pub type ProgramSubscription = (
     PubsubProgramClientSubscription,
@@ -185,6 +192,9 @@ pub type AccountSubscription = (
     Receiver<RpcResponse<UiAccount>>,
 );
 
+pub type PubsubVoteClientSubscription = PubsubClientSubscription<RpcVote>;
+pub type VoteSubscription = (PubsubVoteClientSubscription, Receiver<RpcVote>);
+
 pub type PubsubRootClientSubscription = PubsubClientSubscription<Slot>;
 pub type RootSubscription = (PubsubRootClientSubscription, Receiver<Slot>);
 
@@ -232,7 +242,7 @@ impl PubsubClient {
     ) -> Result<AccountSubscription, PubsubClientError> {
         let url = Url::parse(url)?;
         let socket = connect_with_retry(url)?;
-        let (sender, receiver) = channel();
+        let (sender, receiver) = unbounded();
 
         let socket = Arc::new(RwLock::new(socket));
         let socket_clone = socket.clone();
@@ -266,6 +276,45 @@ impl PubsubClient {
         Ok((result, receiver))
     }
 
+    pub fn block_subscribe(
+        url: &str,
+        filter: RpcBlockSubscribeFilter,
+        config: Option<RpcBlockSubscribeConfig>,
+    ) -> Result<BlockSubscription, PubsubClientError> {
+        let url = Url::parse(url)?;
+        let socket = connect_with_retry(url)?;
+        let (sender, receiver) = unbounded();
+
+        let socket = Arc::new(RwLock::new(socket));
+        let socket_clone = socket.clone();
+        let exit = Arc::new(AtomicBool::new(false));
+        let exit_clone = exit.clone();
+        let body = json!({
+            "jsonrpc":"2.0",
+            "id":1,
+            "method":"blockSubscribe",
+            "params":[filter, config]
+        })
+        .to_string();
+
+        let subscription_id = PubsubBlockClientSubscription::send_subscribe(&socket_clone, body)?;
+
+        let t_cleanup = std::thread::spawn(move || {
+            Self::cleanup_with_sender(exit_clone, &socket_clone, sender)
+        });
+
+        let result = PubsubClientSubscription {
+            message_type: PhantomData,
+            operation: "block",
+            socket,
+            subscription_id,
+            t_cleanup: Some(t_cleanup),
+            exit,
+        };
+
+        Ok((result, receiver))
+    }
+
     pub fn logs_subscribe(
         url: &str,
         filter: RpcTransactionLogsFilter,
@@ -273,7 +322,7 @@ impl PubsubClient {
     ) -> Result<LogsSubscription, PubsubClientError> {
         let url = Url::parse(url)?;
         let socket = connect_with_retry(url)?;
-        let (sender, receiver) = channel();
+        let (sender, receiver) = unbounded();
 
         let socket = Arc::new(RwLock::new(socket));
         let socket_clone = socket.clone();
@@ -312,7 +361,7 @@ impl PubsubClient {
     ) -> Result<ProgramSubscription, PubsubClientError> {
         let url = Url::parse(url)?;
         let socket = connect_with_retry(url)?;
-        let (sender, receiver) = channel();
+        let (sender, receiver) = unbounded();
 
         let socket = Arc::new(RwLock::new(socket));
         let socket_clone = socket.clone();
@@ -346,10 +395,43 @@ impl PubsubClient {
         Ok((result, receiver))
     }
 
+    pub fn vote_subscribe(url: &str) -> Result<VoteSubscription, PubsubClientError> {
+        let url = Url::parse(url)?;
+        let socket = connect_with_retry(url)?;
+        let (sender, receiver) = unbounded();
+
+        let socket = Arc::new(RwLock::new(socket));
+        let socket_clone = socket.clone();
+        let exit = Arc::new(AtomicBool::new(false));
+        let exit_clone = exit.clone();
+        let body = json!({
+            "jsonrpc":"2.0",
+            "id":1,
+            "method":"voteSubscribe",
+        })
+        .to_string();
+        let subscription_id = PubsubVoteClientSubscription::send_subscribe(&socket_clone, body)?;
+
+        let t_cleanup = std::thread::spawn(move || {
+            Self::cleanup_with_sender(exit_clone, &socket_clone, sender)
+        });
+
+        let result = PubsubClientSubscription {
+            message_type: PhantomData,
+            operation: "vote",
+            socket,
+            subscription_id,
+            t_cleanup: Some(t_cleanup),
+            exit,
+        };
+
+        Ok((result, receiver))
+    }
+
     pub fn root_subscribe(url: &str) -> Result<RootSubscription, PubsubClientError> {
         let url = Url::parse(url)?;
         let socket = connect_with_retry(url)?;
-        let (sender, receiver) = channel();
+        let (sender, receiver) = unbounded();
 
         let socket = Arc::new(RwLock::new(socket));
         let socket_clone = socket.clone();
@@ -386,7 +468,7 @@ impl PubsubClient {
     ) -> Result<SignatureSubscription, PubsubClientError> {
         let url = Url::parse(url)?;
         let socket = connect_with_retry(url)?;
-        let (sender, receiver) = channel();
+        let (sender, receiver) = unbounded();
 
         let socket = Arc::new(RwLock::new(socket));
         let socket_clone = socket.clone();
@@ -424,7 +506,7 @@ impl PubsubClient {
     pub fn slot_subscribe(url: &str) -> Result<SlotsSubscription, PubsubClientError> {
         let url = Url::parse(url)?;
         let socket = connect_with_retry(url)?;
-        let (sender, receiver) = channel::<SlotInfo>();
+        let (sender, receiver) = unbounded::<SlotInfo>();
 
         let socket = Arc::new(RwLock::new(socket));
         let socket_clone = socket.clone();

client/src/rpc_client.rs

@@ -43,8 +43,8 @@ use {
         transaction::{self, uses_durable_nonce, Transaction},
     },
     solana_transaction_status::{
-        EncodedConfirmedBlock, EncodedConfirmedTransaction, TransactionStatus, UiConfirmedBlock,
-        UiTransactionEncoding,
+        EncodedConfirmedBlock, EncodedConfirmedTransactionWithStatusMeta, TransactionStatus,
+        UiConfirmedBlock, UiTransactionEncoding,
     },
     solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY,
     std::{
@@ -1329,7 +1329,7 @@ impl RpcClient {
     /// # Ok::<(), ClientError>(())
     /// ```
     pub fn get_highest_snapshot_slot(&self) -> ClientResult<RpcSnapshotSlotInfo> {
-        if self.get_node_version()? < semver::Version::new(1, 8, 0) {
+        if self.get_node_version()? < semver::Version::new(1, 9, 0) {
             #[allow(deprecated)]
             self.get_snapshot_slot().map(|full| RpcSnapshotSlotInfo {
                 full,
@@ -2943,7 +2943,7 @@ impl RpcClient {
         &self,
         signature: &Signature,
         encoding: UiTransactionEncoding,
-    ) -> ClientResult<EncodedConfirmedTransaction> {
+    ) -> ClientResult<EncodedConfirmedTransactionWithStatusMeta> {
         self.send(
             self.maybe_map_request(RpcRequest::GetTransaction)?,
             json!([signature.to_string(), encoding]),
@@ -3006,7 +3006,7 @@ impl RpcClient {
         &self,
         signature: &Signature,
         config: RpcTransactionConfig,
-    ) -> ClientResult<EncodedConfirmedTransaction> {
+    ) -> ClientResult<EncodedConfirmedTransactionWithStatusMeta> {
         self.send(
             self.maybe_map_request(RpcRequest::GetTransaction)?,
             json!([signature.to_string(), config]),
@@ -3022,7 +3022,7 @@ impl RpcClient {
         &self,
         signature: &Signature,
         encoding: UiTransactionEncoding,
-    ) -> ClientResult<EncodedConfirmedTransaction> {
+    ) -> ClientResult<EncodedConfirmedTransactionWithStatusMeta> {
         self.send(
             RpcRequest::GetConfirmedTransaction,
             json!([signature.to_string(), encoding]),
@@ -3038,7 +3038,7 @@ impl RpcClient {
         &self,
         signature: &Signature,
         config: RpcConfirmedTransactionConfig,
-    ) -> ClientResult<EncodedConfirmedTransaction> {
+    ) -> ClientResult<EncodedConfirmedTransactionWithStatusMeta> {
         self.send(
             RpcRequest::GetConfirmedTransaction,
             json!([signature.to_string(), config]),
@@ -4747,7 +4747,7 @@ impl RpcClient {
         commitment: CommitmentConfig,
     ) -> ClientResult<(Hash, u64)> {
         let (blockhash, last_valid_block_height) =
-            if self.get_node_version()? < semver::Version::new(1, 8, 0) {
+            if self.get_node_version()? < semver::Version::new(1, 9, 0) {
                 let Fees {
                     blockhash,
                     last_valid_block_height,
@@ -4781,7 +4781,7 @@ impl RpcClient {
         blockhash: &Hash,
         commitment: CommitmentConfig,
     ) -> ClientResult<bool> {
-        let result = if self.get_node_version()? < semver::Version::new(1, 8, 0) {
+        let result = if self.get_node_version()? < semver::Version::new(1, 9, 0) {
             self.get_fee_calculator_for_blockhash_with_commitment(blockhash, commitment)?
                 .value
                 .is_some()
@@ -4940,6 +4940,7 @@ mod tests {
         super::*,
         crate::{client_error::ClientErrorKind, mock_sender::PUBKEY},
         assert_matches::assert_matches,
+        crossbeam_channel::unbounded,
         jsonrpc_core::{futures::prelude::*, Error, IoHandler, Params},
         jsonrpc_http_server::{AccessControlAllowOrigin, DomainsValidation, ServerBuilder},
         serde_json::Number,
@@ -4949,7 +4950,7 @@ mod tests {
             system_transaction,
             transaction::TransactionError,
         },
-        std::{io, sync::mpsc::channel, thread},
+        std::{io, thread},
     };
 
     #[test]
@@ -4969,7 +4970,7 @@ mod tests {
     }
 
     fn _test_send() {
-        let (sender, receiver) = channel();
+        let (sender, receiver) = unbounded();
         thread::spawn(move || {
             let rpc_addr = "0.0.0.0:0".parse().unwrap();
             let mut io = IoHandler::default();

client/src/rpc_config.rs

@@ -182,6 +182,23 @@ pub struct RpcSignatureSubscribeConfig {
     pub enable_received_notification: Option<bool>,
 }
 
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum RpcBlockSubscribeFilter {
+    All,
+    MentionsAccountOrProgram(String),
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcBlockSubscribeConfig {
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+    pub encoding: Option<UiTransactionEncoding>,
+    pub transaction_details: Option<TransactionDetails>,
+    pub show_rewards: Option<bool>,
+}
+
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcSignaturesForAddressConfig {

client/src/rpc_custom_error.rs

@@ -20,6 +20,7 @@ pub const JSON_RPC_SERVER_ERROR_TRANSACTION_HISTORY_NOT_AVAILABLE: i64 = -32011;
 pub const JSON_RPC_SCAN_ERROR: i64 = -32012;
 pub const JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_LEN_MISMATCH: i64 = -32013;
 pub const JSON_RPC_SERVER_ERROR_BLOCK_STATUS_NOT_AVAILABLE_YET: i64 = -32014;
+pub const JSON_RPC_SERVER_ERROR_UNSUPPORTED_TRANSACTION_VERSION: i64 = -32015;
 
 #[derive(Error, Debug)]
 pub enum RpcCustomError {
@@ -57,6 +58,8 @@ pub enum RpcCustomError {
     TransactionSignatureLenMismatch,
     #[error("BlockStatusNotAvailableYet")]
     BlockStatusNotAvailableYet { slot: Slot },
+    #[error("UnsupportedTransactionVersion")]
+    UnsupportedTransactionVersion,
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -169,6 +172,11 @@ impl From<RpcCustomError> for Error {
                 message: format!("Block status not yet available for slot {}", slot),
                 data: None,
             },
+            RpcCustomError::UnsupportedTransactionVersion => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_UNSUPPORTED_TRANSACTION_VERSION),
+                message: "Versioned transactions are not supported".to_string(),
+                data: None,
+            },
         }
     }
 }

client/src/rpc_response.rs

@@ -9,9 +9,10 @@ use {
         transaction::{Result, TransactionError},
     },
     solana_transaction_status::{
-        ConfirmedTransactionStatusWithSignature, TransactionConfirmationStatus,
+        ConfirmedTransactionStatusWithSignature, TransactionConfirmationStatus, UiConfirmedBlock,
     },
     std::{collections::HashMap, fmt, net::SocketAddr},
+    thiserror::Error,
 };
 
 pub type RpcResult<T> = client_error::Result<Response<T>>;
@@ -424,6 +425,23 @@ pub struct RpcInflationReward {
     pub commission: Option<u8>, // Vote account commission when the reward was credited
 }
 
+#[derive(Clone, Deserialize, Serialize, Debug, Error, Eq, PartialEq)]
+pub enum RpcBlockUpdateError {
+    #[error("block store error")]
+    BlockStoreError,
+
+    #[error("unsupported transaction version")]
+    UnsupportedTransactionVersion,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcBlockUpdate {
+    pub slot: Slot,
+    pub block: Option<UiConfirmedBlock>,
+    pub err: Option<RpcBlockUpdateError>,
+}
+
 impl From<ConfirmedTransactionStatusWithSignature> for RpcConfirmedTransactionStatusWithSignature {
     fn from(value: ConfirmedTransactionStatusWithSignature) -> Self {
         let ConfirmedTransactionStatusWithSignature {

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.9.0"
+version = "1.10.0"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-core"
 readme = "../README.md"
@@ -21,43 +21,43 @@ bs58 = "0.4.0"
 chrono = { version = "0.4.11", features = ["serde"] }
 crossbeam-channel = "0.5"
 dashmap = { version = "4.0.2", features = ["rayon", "raw-api"] }
-etcd-client = { version = "0.8.1", features = ["tls"]}
+etcd-client = { version = "0.8.3", features = ["tls"]}
 fs_extra = "1.2.0"
 histogram = "0.6.9"
-itertools = "0.10.1"
+itertools = "0.10.3"
 log = "0.4.14"
-lru = "0.7.0"
+lru = "0.7.2"
 rand = "0.7.0"
 rand_chacha = "0.2.2"
-raptorq = "1.6.4"
 rayon = "1.5.1"
 retain_mut = "0.1.5"
-serde = "1.0.130"
+serde = "1.0.133"
 serde_derive = "1.0.103"
-solana-accountsdb-plugin-manager = { path = "../accountsdb-plugin-manager", version = "=1.9.0" }
-solana-client = { path = "../client", version = "=1.9.0" }
-solana-entry = { path = "../entry", version = "=1.9.0" }
-solana-gossip = { path = "../gossip", version = "=1.9.0" }
-solana-ledger = { path = "../ledger", version = "=1.9.0" }
-solana-logger = { path = "../logger", version = "=1.9.0" }
-solana-measure = { path = "../measure", version = "=1.9.0" }
-solana-metrics = { path = "../metrics", version = "=1.9.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.9.0" }
-solana-perf = { path = "../perf", version = "=1.9.0" }
-solana-poh = { path = "../poh", version = "=1.9.0" }
-solana-rpc = { path = "../rpc", version = "=1.9.0" }
-solana-replica-lib = { path = "../replica-lib", version = "=1.9.0" }
-solana-runtime = { path = "../runtime", version = "=1.9.0" }
-solana-sdk = { path = "../sdk", version = "=1.9.0" }
-solana-frozen-abi = { path = "../frozen-abi", version = "=1.9.0" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.9.0" }
-solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.9.0" }
-solana-streamer = { path = "../streamer", version = "=1.9.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.9.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.9.0" }
-tempfile = "3.2.0"
+solana-address-lookup-table-program = { path = "../programs/address-lookup-table", version = "=1.10.0" }
+solana-accountsdb-plugin-manager = { path = "../accountsdb-plugin-manager", version = "=1.10.0" }
+solana-client = { path = "../client", version = "=1.10.0" }
+solana-entry = { path = "../entry", version = "=1.10.0" }
+solana-gossip = { path = "../gossip", version = "=1.10.0" }
+solana-ledger = { path = "../ledger", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-metrics = { path = "../metrics", version = "=1.10.0" }
+solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
+solana-perf = { path = "../perf", version = "=1.10.0" }
+solana-poh = { path = "../poh", version = "=1.10.0" }
+solana-program-runtime = { path = "../program-runtime", version = "=1.10.0" }
+solana-rpc = { path = "../rpc", version = "=1.10.0" }
+solana-replica-lib = { path = "../replica-lib", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.10.0" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.10.0" }
+solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.10.0" }
+solana-streamer = { path = "../streamer", version = "=1.10.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
+solana-vote-program = { path = "../programs/vote", version = "=1.10.0" }
+tempfile = "3.3.0"
 thiserror = "1.0"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.9.0" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "=1.10.0" }
 sys-info = "0.9.1"
 tokio = { version = "1", features = ["full"] }
 trees = "0.4.2"
@@ -68,12 +68,14 @@ jsonrpc-core-client = { version = "18.0.0", features = ["ipc", "ws"] }
 jsonrpc-derive = "18.0.0"
 jsonrpc-pubsub = "18.0.0"
 matches = "0.1.9"
+raptorq = "1.6.4"
 reqwest = { version = "0.11.6", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-serde_json = "1.0.72"
+serde_json = "1.0.74"
 serial_test = "0.5.1"
-solana-program-runtime = { path = "../program-runtime", version = "=1.9.0" }
-solana-stake-program = { path = "../programs/stake", version = "=1.9.0" }
-solana-version = { path = "../version", version = "=1.9.0" }
+solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-program-runtime = { path = "../program-runtime", version = "=1.10.0" }
+solana-stake-program = { path = "../programs/stake", version = "=1.10.0" }
+solana-version = { path = "../version", version = "=1.10.0" }
 static_assertions = "1.1.0"
 systemstat = "0.1.10"
 

core/benches/banking_stage.rs

@@ -4,12 +4,13 @@
 extern crate test;
 
 use {
-    crossbeam_channel::unbounded,
+    crossbeam_channel::{unbounded, Receiver},
     log::*,
     rand::{thread_rng, Rng},
     rayon::prelude::*,
     solana_core::{
         banking_stage::{BankingStage, BankingStageStats},
+        packet_deduper::PacketDeduper,
         qos_service::QosService,
     },
     solana_entry::entry::{next_hash, Entry},
@@ -20,7 +21,7 @@ use {
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
         get_tmp_ledger_path,
     },
-    solana_perf::{packet::to_packets_chunked, test_tx::test_tx},
+    solana_perf::{packet::to_packet_batches, test_tx::test_tx},
     solana_poh::poh_recorder::{create_test_recorder, WorkingBankEntry},
     solana_runtime::{bank::Bank, cost_model::CostModel},
     solana_sdk::{
@@ -36,7 +37,7 @@ use {
     solana_streamer::socket::SocketAddrSpace,
     std::{
         collections::VecDeque,
-        sync::{atomic::Ordering, mpsc::Receiver, Arc, RwLock},
+        sync::{atomic::Ordering, Arc, RwLock},
         time::{Duration, Instant},
     },
     test::Bencher,
@@ -77,11 +78,11 @@ fn bench_consume_buffered(bencher: &mut Bencher) {
         let tx = test_tx();
         let len = 4096;
         let chunk_size = 1024;
-        let batches = to_packets_chunked(&vec![tx; len], chunk_size);
-        let mut packets = VecDeque::new();
+        let batches = to_packet_batches(&vec![tx; len], chunk_size);
+        let mut packet_batches = VecDeque::new();
         for batch in batches {
             let batch_len = batch.packets.len();
-            packets.push_back((batch, vec![0usize; batch_len], false));
+            packet_batches.push_back((batch, vec![0usize; batch_len], false));
         }
         let (s, _r) = unbounded();
         // This tests the performance of buffering packets.
@@ -91,13 +92,13 @@ fn bench_consume_buffered(bencher: &mut Bencher) {
                 &my_pubkey,
                 std::u128::MAX,
                 &poh_recorder,
-                &mut packets,
+                &mut packet_batches,
                 None,
                 &s,
                 None::<Box<dyn Fn()>>,
                 &BankingStageStats::default(),
                 &recorder,
-                &Arc::new(QosService::new(Arc::new(RwLock::new(CostModel::default())))),
+                &QosService::new(Arc::new(RwLock::new(CostModel::default())), 1),
             );
         });
 
@@ -168,13 +169,13 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) {
     let (vote_sender, vote_receiver) = unbounded();
     let mut bank = Bank::new_for_benches(&genesis_config);
     // Allow arbitrary transaction processing time for the purposes of this bench
-    bank.ns_per_slot = std::u128::MAX;
-    let bank = Arc::new(Bank::new_for_benches(&genesis_config));
+    bank.ns_per_slot = u128::MAX;
+    let bank = Arc::new(bank);
 
     // set cost tracker limits to MAX so it will not filter out TXs
     bank.write_cost_tracker()
         .unwrap()
-        .set_limits(std::u64::MAX, std::u64::MAX);
+        .set_limits(std::u64::MAX, std::u64::MAX, std::u64::MAX);
 
     debug!("threads: {} txs: {}", num_threads, txes);
 
@@ -206,7 +207,7 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) {
         assert!(r.is_ok(), "sanity parallel execution");
     }
     bank.clear_signatures();
-    let verified: Vec<_> = to_packets_chunked(&transactions, PACKETS_PER_BATCH);
+    let verified: Vec<_> = to_packet_batches(&transactions, PACKETS_PER_BATCH);
     let ledger_path = get_tmp_ledger_path!();
     {
         let blockstore = Arc::new(
@@ -221,6 +222,7 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) {
         );
         let cluster_info = Arc::new(cluster_info);
         let (s, _r) = unbounded();
+        let packet_deduper = PacketDeduper::default();
         let _banking_stage = BankingStage::new(
             &cluster_info,
             &poh_recorder,
@@ -230,6 +232,7 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) {
             None,
             s,
             Arc::new(RwLock::new(CostModel::default())),
+            packet_deduper.clone(),
         );
         poh_recorder.lock().unwrap().set_bank(&bank);
 
@@ -264,6 +267,7 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) {
             // in this chunk, but since we rotate between CHUNKS then
             // we should clear them by the time we come around again to re-use that chunk.
             bank.clear_signatures();
+            packet_deduper.reset();
             trace!(
                 "time: {} checked: {} sent: {}",
                 duration_as_us(&now.elapsed()),

core/benches/retransmit_stage.rs

@@ -4,6 +4,7 @@ extern crate solana_core;
 extern crate test;
 
 use {
+    crossbeam_channel::unbounded,
     log::*,
     solana_core::retransmit_stage::retransmitter,
     solana_entry::entry::Entry,
@@ -30,7 +31,6 @@ use {
         net::UdpSocket,
         sync::{
             atomic::{AtomicUsize, Ordering},
-            mpsc::channel,
             Arc, RwLock,
         },
         thread::{sleep, Builder},
@@ -77,7 +77,7 @@ fn bench_retransmitter(bencher: &mut Bencher) {
     let bank_forks = BankForks::new(bank0);
     let bank = bank_forks.working_bank();
     let bank_forks = Arc::new(RwLock::new(bank_forks));
-    let (shreds_sender, shreds_receiver) = channel();
+    let (shreds_sender, shreds_receiver) = unbounded();
     const NUM_THREADS: usize = 2;
     let sockets = (0..NUM_THREADS)
         .map(|_| UdpSocket::bind("0.0.0.0:0").unwrap())
@@ -100,7 +100,11 @@ fn bench_retransmitter(bencher: &mut Bencher) {
     let slot = 0;
     let parent = 0;
     let shredder = Shredder::new(slot, parent, 0, 0).unwrap();
-    let mut data_shreds = shredder.entries_to_shreds(&keypair, &entries, true, 0).0;
+    let (mut data_shreds, _) = shredder.entries_to_shreds(
+        &keypair, &entries, true, // is_last_in_slot
+        0,    // next_shred_index
+        0,    // next_code_index
+    );
 
     let num_packets = data_shreds.len();
 

core/benches/shredder.rs

@@ -40,16 +40,14 @@ fn make_shreds(num_shreds: usize) -> Vec<Shred> {
     );
     let entries = make_large_unchained_entries(txs_per_entry, num_entries);
     let shredder = Shredder::new(1, 0, 0, 0).unwrap();
-    let data_shreds = shredder
-        .entries_to_data_shreds(
-            &Keypair::new(),
-            &entries,
-            true, // is_last_in_slot
-            0,    // next_shred_index
-            0,    // fec_set_offset
-            &mut ProcessShredsStats::default(),
-        )
-        .0;
+    let data_shreds = shredder.entries_to_data_shreds(
+        &Keypair::new(),
+        &entries,
+        true, // is_last_in_slot
+        0,    // next_shred_index
+        0,    // fec_set_offset
+        &mut ProcessShredsStats::default(),
+    );
     assert!(data_shreds.len() >= num_shreds);
     data_shreds
 }
@@ -76,7 +74,7 @@ fn bench_shredder_ticks(bencher: &mut Bencher) {
     let entries = create_ticks(num_ticks, 0, Hash::default());
     bencher.iter(|| {
         let shredder = Shredder::new(1, 0, 0, 0).unwrap();
-        shredder.entries_to_shreds(&kp, &entries, true, 0);
+        shredder.entries_to_shreds(&kp, &entries, true, 0, 0);
     })
 }
 
@@ -95,7 +93,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
     // 1Mb
     bencher.iter(|| {
         let shredder = Shredder::new(1, 0, 0, 0).unwrap();
-        shredder.entries_to_shreds(&kp, &entries, true, 0);
+        shredder.entries_to_shreds(&kp, &entries, true, 0, 0);
     })
 }
 
@@ -108,7 +106,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
     let num_ticks = max_ticks_per_n_shreds(1, Some(shred_size)) * num_shreds as u64;
     let entries = create_ticks(num_ticks, 0, Hash::default());
     let shredder = Shredder::new(1, 0, 0, 0).unwrap();
-    let data_shreds = shredder.entries_to_shreds(&kp, &entries, true, 0).0;
+    let (data_shreds, _) = shredder.entries_to_shreds(&kp, &entries, true, 0, 0);
     bencher.iter(|| {
         let raw = &mut Shredder::deshred(&data_shreds).unwrap();
         assert_ne!(raw.len(), 0);
@@ -135,6 +133,7 @@ fn bench_shredder_coding(bencher: &mut Bencher) {
         Shredder::generate_coding_shreds(
             &data_shreds[..symbol_count],
             true, // is_last_in_slot
+            0,    // next_code_index
         )
         .len();
     })
@@ -147,6 +146,7 @@ fn bench_shredder_decoding(bencher: &mut Bencher) {
     let coding_shreds = Shredder::generate_coding_shreds(
         &data_shreds[..symbol_count],
         true, // is_last_in_slot
+        0,    // next_code_index
     );
     bencher.iter(|| {
         Shredder::try_recovery(coding_shreds[..].to_vec()).unwrap();

core/benches/sigverify_stage.rs

@@ -1,4 +1,5 @@
 #![feature(test)]
+#![allow(clippy::integer_arithmetic)]
 
 extern crate solana_core;
 extern crate test;
@@ -8,36 +9,32 @@ use {
     log::*,
     rand::{thread_rng, Rng},
     solana_core::{sigverify::TransactionSigVerifier, sigverify_stage::SigVerifyStage},
-    solana_perf::{packet::to_packets_chunked, test_tx::test_tx},
+    solana_perf::{packet::to_packet_batches, test_tx::test_tx},
     solana_sdk::{
         hash::Hash,
         signature::{Keypair, Signer},
         system_transaction,
         timing::duration_as_ms,
     },
-    std::{
-        sync::mpsc::channel,
-        time::{Duration, Instant},
-    },
+    std::time::{Duration, Instant},
     test::Bencher,
 };
 
-#[bench]
-fn bench_packet_discard(bencher: &mut Bencher) {
+fn run_bench_packet_discard(num_ips: usize, bencher: &mut Bencher) {
     solana_logger::setup();
     let len = 30 * 1000;
     let chunk_size = 1024;
     let tx = test_tx();
-    let mut batches = to_packets_chunked(&vec![tx; len], chunk_size);
+    let mut batches = to_packet_batches(&vec![tx; len], chunk_size);
 
     let mut total = 0;
 
-    let ips: Vec<_> = (0..10_000)
+    let ips: Vec<_> = (0..num_ips)
         .into_iter()
         .map(|_| {
             let mut addr = [0u16; 8];
             thread_rng().fill(&mut addr);
-            addr
+            std::net::IpAddr::from(addr)
         })
         .collect();
 
@@ -54,16 +51,26 @@ fn bench_packet_discard(bencher: &mut Bencher) {
         SigVerifyStage::discard_excess_packets(&mut batches, 10_000);
         for batch in batches.iter_mut() {
             for p in batch.packets.iter_mut() {
-                p.meta.discard = false;
+                p.meta.set_discard(false);
             }
         }
     });
 }
 
+#[bench]
+fn bench_packet_discard_many_senders(bencher: &mut Bencher) {
+    run_bench_packet_discard(1000, bencher);
+}
+
+#[bench]
+fn bench_packet_discard_single_sender(bencher: &mut Bencher) {
+    run_bench_packet_discard(1, bencher);
+}
+
 #[bench]
 fn bench_sigverify_stage(bencher: &mut Bencher) {
     solana_logger::setup();
-    let (packet_s, packet_r) = channel();
+    let (packet_s, packet_r) = unbounded();
     let (verified_s, verified_r) = unbounded();
     let verifier = TransactionSigVerifier::default();
     let stage = SigVerifyStage::new(packet_r, verified_s, verifier);
@@ -74,7 +81,7 @@ fn bench_sigverify_stage(bencher: &mut Bencher) {
     let chunk_size = 1024;
     let mut batches = if use_same_tx {
         let tx = test_tx();
-        to_packets_chunked(&vec![tx; len], chunk_size)
+        to_packet_batches(&vec![tx; len], chunk_size)
     } else {
         let from_keypair = Keypair::new();
         let to_keypair = Keypair::new();
@@ -89,7 +96,7 @@ fn bench_sigverify_stage(bencher: &mut Bencher) {
                 )
             })
             .collect();
-        to_packets_chunked(&txs, chunk_size)
+        to_packet_batches(&txs, chunk_size)
     };
 
     trace!(

core/src/accounts_hash_verifier.rs

@@ -5,6 +5,7 @@
 // set and halt the node if a mismatch is detected.
 
 use {
+    crossbeam_channel::RecvTimeoutError,
     rayon::ThreadPool,
     solana_gossip::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES},
     solana_measure::measure::Measure,
@@ -24,7 +25,6 @@ use {
         path::{Path, PathBuf},
         sync::{
             atomic::{AtomicBool, Ordering},
-            mpsc::RecvTimeoutError,
             Arc,
         },
         thread::{self, Builder, JoinHandle},

core/src/ancestor_hashes_service.rs

@@ -14,7 +14,7 @@ use {
     solana_ledger::{blockstore::Blockstore, shred::SIZE_OF_NONCE},
     solana_measure::measure::Measure,
     solana_perf::{
-        packet::{limited_deserialize, Packet, Packets},
+        packet::{limited_deserialize, Packet, PacketBatch},
         recycler::Recycler,
     },
     solana_runtime::bank::Bank,
@@ -23,13 +23,12 @@ use {
         pubkey::Pubkey,
         timing::timestamp,
     },
-    solana_streamer::streamer::{self, PacketReceiver},
+    solana_streamer::streamer::{self, PacketBatchReceiver},
     std::{
         collections::HashSet,
         net::UdpSocket,
         sync::{
             atomic::{AtomicBool, Ordering},
-            mpsc::channel,
             Arc, RwLock,
         },
         thread::{self, sleep, Builder, JoinHandle},
@@ -147,7 +146,7 @@ impl AncestorHashesService {
     ) -> Self {
         let outstanding_requests: Arc<RwLock<OutstandingAncestorHashesRepairs>> =
             Arc::new(RwLock::new(OutstandingAncestorHashesRepairs::default()));
-        let (response_sender, response_receiver) = channel();
+        let (response_sender, response_receiver) = unbounded();
         let t_receiver = streamer::receiver(
             ancestor_hashes_request_socket.clone(),
             &exit,
@@ -197,7 +196,7 @@ impl AncestorHashesService {
     /// Listen for responses to our ancestors hashes repair requests
     fn run_responses_listener(
         ancestor_hashes_request_statuses: Arc<DashMap<Slot, DeadSlotAncestorRequestStatus>>,
-        response_receiver: PacketReceiver,
+        response_receiver: PacketBatchReceiver,
         blockstore: Arc<Blockstore>,
         outstanding_requests: Arc<RwLock<OutstandingAncestorHashesRepairs>>,
         exit: Arc<AtomicBool>,
@@ -240,7 +239,7 @@ impl AncestorHashesService {
     /// Process messages from the network
     fn process_new_packets_from_channel(
         ancestor_hashes_request_statuses: &DashMap<Slot, DeadSlotAncestorRequestStatus>,
-        response_receiver: &PacketReceiver,
+        response_receiver: &PacketBatchReceiver,
         blockstore: &Blockstore,
         outstanding_requests: &RwLock<OutstandingAncestorHashesRepairs>,
         stats: &mut AncestorHashesResponsesStats,
@@ -249,17 +248,17 @@ impl AncestorHashesService {
         retryable_slots_sender: &RetryableSlotsSender,
     ) -> Result<()> {
         let timeout = Duration::new(1, 0);
-        let mut responses = vec![response_receiver.recv_timeout(timeout)?];
-        let mut total_packets = responses[0].packets.len();
+        let mut packet_batches = vec![response_receiver.recv_timeout(timeout)?];
+        let mut total_packets = packet_batches[0].packets.len();
 
         let mut dropped_packets = 0;
-        while let Ok(more) = response_receiver.try_recv() {
-            total_packets += more.packets.len();
+        while let Ok(batch) = response_receiver.try_recv() {
+            total_packets += batch.packets.len();
             if total_packets < *max_packets {
                 // Drop the rest in the channel in case of DOS
-                responses.push(more);
+                packet_batches.push(batch);
             } else {
-                dropped_packets += more.packets.len();
+                dropped_packets += batch.packets.len();
             }
         }
 
@@ -267,10 +266,10 @@ impl AncestorHashesService {
         stats.total_packets += total_packets;
 
         let mut time = Measure::start("ancestor_hashes::handle_packets");
-        for response in responses {
-            Self::process_single_packets(
+        for packet_batch in packet_batches {
+            Self::process_packet_batch(
                 ancestor_hashes_request_statuses,
-                response,
+                packet_batch,
                 stats,
                 outstanding_requests,
                 blockstore,
@@ -289,16 +288,16 @@ impl AncestorHashesService {
         Ok(())
     }
 
-    fn process_single_packets(
+    fn process_packet_batch(
         ancestor_hashes_request_statuses: &DashMap<Slot, DeadSlotAncestorRequestStatus>,
-        packets: Packets,
+        packet_batch: PacketBatch,
         stats: &mut AncestorHashesResponsesStats,
         outstanding_requests: &RwLock<OutstandingAncestorHashesRepairs>,
         blockstore: &Blockstore,
         duplicate_slots_reset_sender: &DuplicateSlotsResetSender,
         retryable_slots_sender: &RetryableSlotsSender,
     ) {
-        packets.packets.iter().for_each(|packet| {
+        packet_batch.packets.iter().for_each(|packet| {
             let decision = Self::verify_and_process_ancestor_response(
                 packet,
                 ancestor_hashes_request_statuses,
@@ -328,7 +327,7 @@ impl AncestorHashesService {
         blockstore: &Blockstore,
     ) -> Option<(Slot, DuplicateAncestorDecision)> {
         let from_addr = packet.meta.addr();
-        limited_deserialize(&packet.data[..packet.meta.size - SIZE_OF_NONCE])
+        limited_deserialize(&packet.data[..packet.meta.size.saturating_sub(SIZE_OF_NONCE)])
             .ok()
             .and_then(|ancestor_hashes_response| {
                 // Verify the response
@@ -703,7 +702,7 @@ mod test {
         solana_runtime::{accounts_background_service::AbsRequestSender, bank_forks::BankForks},
         solana_sdk::{hash::Hash, signature::Keypair},
         solana_streamer::socket::SocketAddrSpace,
-        std::{collections::HashMap, sync::mpsc::channel},
+        std::collections::HashMap,
         trees::tr,
     };
 
@@ -871,7 +870,7 @@ mod test {
         t_listen: JoinHandle<()>,
         exit: Arc<AtomicBool>,
         responder_info: ContactInfo,
-        response_receiver: PacketReceiver,
+        response_receiver: PacketBatchReceiver,
         correct_bank_hashes: HashMap<Slot, Hash>,
     }
 
@@ -896,8 +895,8 @@ mod test {
             // Set up thread to give us responses
             let ledger_path = get_tmp_ledger_path!();
             let exit = Arc::new(AtomicBool::new(false));
-            let (requests_sender, requests_receiver) = channel();
-            let (response_sender, response_receiver) = channel();
+            let (requests_sender, requests_receiver) = unbounded();
+            let (response_sender, response_receiver) = unbounded();
 
             // Set up blockstore for responses
             let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
@@ -1033,15 +1032,6 @@ mod test {
             is_frozen,
         );
 
-        /*{
-            let w_bank_forks = bank_forks.write().unwrap();
-            assert!(w_bank_forks.get(dead_slot).is_none());
-            let parent = w_bank_forks.get(dead_slot - 1).unwrap().clone();
-            let dead_bank = Bank::new_from_parent(&parent, &Pubkey::default(), dead_slot);
-            bank_forks.insert(dead_bank);
-
-        }*/
-
         // Create slots [slot, slot + num_ancestors) with 5 shreds apiece
         let (shreds, _) = make_many_slot_entries(dead_slot, dead_slot, 5);
         blockstore
@@ -1369,6 +1359,34 @@ mod test {
         assert!(ancestor_hashes_request_statuses.is_empty());
     }
 
+    #[test]
+    fn test_verify_and_process_ancestor_responses_invalid_packet() {
+        let bank0 = Bank::default_for_tests();
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank0)));
+
+        let ManageAncestorHashesState {
+            ancestor_hashes_request_statuses,
+            outstanding_requests,
+            ..
+        } = ManageAncestorHashesState::new(bank_forks);
+
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Blockstore::open(&ledger_path).unwrap();
+
+        // Create invalid packet with fewer bytes than the size of the nonce
+        let mut packet = Packet::default();
+        packet.meta.size = 0;
+
+        assert!(AncestorHashesService::verify_and_process_ancestor_response(
+            &packet,
+            &ancestor_hashes_request_statuses,
+            &mut AncestorHashesResponsesStats::default(),
+            &outstanding_requests,
+            &blockstore,
+        )
+        .is_none());
+    }
+
     #[test]
     fn test_ancestor_hashes_service_manage_ancestor_hashes_after_replay_dump() {
         let dead_slot = MAX_ANCESTOR_RESPONSES as Slot;