Update to spl-token 2

* Update to token pack/unpack changes (#11900)

(cherry picked from commit 2eff9a19c3)

# Conflicts:
#	account-decoder/Cargo.toml
#	core/Cargo.toml

* Fix conflicts

Co-authored-by: Tyera Eulberg <teulberg@gmail.com>
Co-authored-by: Tyera Eulberg <tyera@solana.com>

.buildkite/hooks/post-command

@@ -3,16 +3,19 @@
 #
 # Save target/ for the next CI build on this machine
 #
-(
-  set -x
-  d=$HOME/cargo-target-cache/"$BUILDKITE_LABEL"
-  mkdir -p "$d"
-  set -x
-  rsync -a --delete --link-dest="$PWD" target "$d"
-  du -hs "$d"
-  read -r cacheSizeInGB _ < <(du -s --block-size=1800000000 "$d")
-  echo "--- ${cacheSizeInGB}GB: $d"
-)
+if [[ -z $CARGO_TARGET_CACHE ]]; then
+  echo "+++ CARGO_TARGET_CACHE not defined" # pre-command should have defined it
+else
+  (
+    set -x
+    mkdir -p "$CARGO_TARGET_CACHE"
+    set -x
+    rsync -a --delete --link-dest="$PWD" target "$CARGO_TARGET_CACHE"
+    du -hs "$CARGO_TARGET_CACHE"
+    read -r cacheSizeInGB _ < <(du -s --block-size=1800000000 "$CARGO_TARGET_CACHE")
+    echo "--- ${cacheSizeInGB}GB: $CARGO_TARGET_CACHE"
+  )
+fi
 
 #
 # Add job_stats data point

.buildkite/hooks/pre-command

@@ -11,23 +11,24 @@ export PS4="++"
 #
 # Restore target/ from the previous CI build on this machine
 #
+eval "$(ci/channel-info.sh)"
+export CARGO_TARGET_CACHE=$HOME/cargo-target-cache/"$CHANNEL"-"$BUILDKITE_LABEL"
 (
   set -x
-  d=$HOME/cargo-target-cache/"$BUILDKITE_LABEL"
   MAX_CACHE_SIZE=18 # gigabytes
 
-  if [[ -d $d ]]; then
-    du -hs "$d"
-    read -r cacheSizeInGB _ < <(du -s --block-size=1800000000 "$d")
-    echo "--- ${cacheSizeInGB}GB: $d"
+  if [[ -d $CARGO_TARGET_CACHE ]]; then
+    du -hs "$CARGO_TARGET_CACHE"
+    read -r cacheSizeInGB _ < <(du -s --block-size=1800000000 "$CARGO_TARGET_CACHE")
+    echo "--- ${cacheSizeInGB}GB: $CARGO_TARGET_CACHE"
     if [[ $cacheSizeInGB -gt $MAX_CACHE_SIZE ]]; then
-      echo "--- $d is too large, removing it"
-      rm -rf "$d"
+      echo "--- $CARGO_TARGET_CACHE is too large, removing it"
+      rm -rf "$CARGO_TARGET_CACHE"
     fi
   else
-    echo "--- $d not present"
+    echo "--- $CARGO_TARGET_CACHE not present"
   fi
 
-  mkdir -p "$d"/target
-  rsync -a --delete --link-dest="$d" "$d"/target .
+  mkdir -p "$CARGO_TARGET_CACHE"/target
+  rsync -a --delete --link-dest="$CARGO_TARGET_CACHE" "$CARGO_TARGET_CACHE"/target .
 )

Cargo.toml

@@ -5,6 +5,9 @@ members = [
     "bench-tps",
     "accounts-bench",
     "banking-bench",
+    "banks-client",
+    "banks-interface",
+    "banks-server",
     "clap-utils",
     "cli-config",
     "client",

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -15,14 +15,14 @@ bs58 = "0.3.1"
 bv = "0.11.1"
 Inflector = "0.11.4"
 lazy_static = "1.4.0"
-solana-config-program = { path = "../programs/config", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-stake-program = { path = "../programs/stake", version = "1.3.4" }
-solana-vote-program = { path = "../programs/vote", version = "1.3.4" }
-spl-token-v1-0 = { package = "spl-token", version = "1.0.6", features = ["skip-no-mangle"] }
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
+solana-config-program = { path = "../programs/config", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-stake-program = { path = "../programs/stake", version = "1.3.6" }
+solana-vote-program = { path = "../programs/vote", version = "1.3.6" }
+spl-token-v2-0 = { package = "spl-token", version = "2.0.3", features = ["skip-no-mangle"] }
 thiserror = "1.0"
 
 [package.metadata.docs.rs]

account-decoder/src/parse_account_data.rs

@@ -3,7 +3,7 @@ use crate::{
     parse_nonce::parse_nonce,
     parse_stake::parse_stake,
     parse_sysvar::parse_sysvar,
-    parse_token::{parse_token, spl_token_id_v1_0},
+    parse_token::{parse_token, spl_token_id_v2_0},
     parse_vote::parse_vote,
 };
 use inflector::Inflector;
@@ -17,7 +17,7 @@ lazy_static! {
     static ref STAKE_PROGRAM_ID: Pubkey = solana_stake_program::id();
     static ref SYSTEM_PROGRAM_ID: Pubkey = system_program::id();
     static ref SYSVAR_PROGRAM_ID: Pubkey = sysvar::id();
-    static ref TOKEN_PROGRAM_ID: Pubkey = spl_token_id_v1_0();
+    static ref TOKEN_PROGRAM_ID: Pubkey = spl_token_id_v2_0();
     static ref VOTE_PROGRAM_ID: Pubkey = solana_vote_program::id();
     pub static ref PARSABLE_PROGRAM_IDS: HashMap<Pubkey, ParsableAccount> = {
         let mut m = HashMap::new();

account-decoder/src/parse_token.rs

@@ -3,32 +3,32 @@ use crate::{
     StringAmount,
 };
 use solana_sdk::pubkey::Pubkey;
-use spl_token_v1_0::{
+use spl_token_v2_0::{
     option::COption,
+    pack::Pack,
     solana_sdk::pubkey::Pubkey as SplTokenPubkey,
-    state::{unpack, Account, Mint, Multisig},
+    state::{Account, AccountState, Mint, Multisig},
 };
-use std::{mem::size_of, str::FromStr};
+use std::str::FromStr;
 
-// A helper function to convert spl_token_v1_0::id() as spl_sdk::pubkey::Pubkey to
+// A helper function to convert spl_token_v2_0::id() as spl_sdk::pubkey::Pubkey to
 // solana_sdk::pubkey::Pubkey
-pub fn spl_token_id_v1_0() -> Pubkey {
-    Pubkey::from_str(&spl_token_v1_0::id().to_string()).unwrap()
+pub fn spl_token_id_v2_0() -> Pubkey {
+    Pubkey::from_str(&spl_token_v2_0::id().to_string()).unwrap()
 }
 
-// A helper function to convert spl_token_v1_0::native_mint::id() as spl_sdk::pubkey::Pubkey to
+// A helper function to convert spl_token_v2_0::native_mint::id() as spl_sdk::pubkey::Pubkey to
 // solana_sdk::pubkey::Pubkey
-pub fn spl_token_v1_0_native_mint() -> Pubkey {
-    Pubkey::from_str(&spl_token_v1_0::native_mint::id().to_string()).unwrap()
+pub fn spl_token_v2_0_native_mint() -> Pubkey {
+    Pubkey::from_str(&spl_token_v2_0::native_mint::id().to_string()).unwrap()
 }
 
 pub fn parse_token(
     data: &[u8],
     mint_decimals: Option<u8>,
 ) -> Result<TokenAccountType, ParseAccountError> {
-    let mut data = data.to_vec();
-    if data.len() == size_of::<Account>() {
-        let account: Account = *unpack(&mut data)
+    if data.len() == Account::get_packed_len() {
+        let account = Account::unpack(data)
             .map_err(|_| ParseAccountError::AccountNotParsable(ParsableAccount::SplToken))?;
         let decimals = mint_decimals.ok_or_else(|| {
             ParseAccountError::AdditionalDataMissing(
@@ -43,8 +43,12 @@ pub fn parse_token(
                 COption::Some(pubkey) => Some(pubkey.to_string()),
                 COption::None => None,
             },
-            is_initialized: account.is_initialized,
-            is_native: account.is_native,
+            state: account.state.into(),
+            is_native: account.is_native(),
+            rent_exempt_reserve: match account.is_native {
+                COption::Some(reserve) => Some(token_amount_to_ui_amount(reserve, decimals)),
+                COption::None => None,
+            },
             delegated_amount: if account.delegate.is_none() {
                 None
             } else {
@@ -53,20 +57,29 @@ pub fn parse_token(
                     decimals,
                 ))
             },
-        }))
-    } else if data.len() == size_of::<Mint>() {
-        let mint: Mint = *unpack(&mut data)
-            .map_err(|_| ParseAccountError::AccountNotParsable(ParsableAccount::SplToken))?;
-        Ok(TokenAccountType::Mint(UiMint {
-            owner: match mint.owner {
+            close_authority: match account.close_authority {
                 COption::Some(pubkey) => Some(pubkey.to_string()),
                 COption::None => None,
             },
+        }))
+    } else if data.len() == Mint::get_packed_len() {
+        let mint = Mint::unpack(data)
+            .map_err(|_| ParseAccountError::AccountNotParsable(ParsableAccount::SplToken))?;
+        Ok(TokenAccountType::Mint(UiMint {
+            mint_authority: match mint.mint_authority {
+                COption::Some(pubkey) => Some(pubkey.to_string()),
+                COption::None => None,
+            },
+            supply: mint.supply.to_string(),
             decimals: mint.decimals,
             is_initialized: mint.is_initialized,
+            freeze_authority: match mint.freeze_authority {
+                COption::Some(pubkey) => Some(pubkey.to_string()),
+                COption::None => None,
+            },
         }))
-    } else if data.len() == size_of::<Multisig>() {
-        let multisig: Multisig = *unpack(&mut data)
+    } else if data.len() == Multisig::get_packed_len() {
+        let multisig = Multisig::unpack(data)
             .map_err(|_| ParseAccountError::AccountNotParsable(ParsableAccount::SplToken))?;
         Ok(TokenAccountType::Multisig(UiMultisig {
             num_required_signers: multisig.m,
@@ -107,10 +120,32 @@ pub struct UiTokenAccount {
     pub token_amount: UiTokenAmount,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub delegate: Option<String>,
-    pub is_initialized: bool,
+    pub state: UiAccountState,
     pub is_native: bool,
     #[serde(skip_serializing_if = "Option::is_none")]
+    pub rent_exempt_reserve: Option<UiTokenAmount>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub delegated_amount: Option<UiTokenAmount>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub close_authority: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub enum UiAccountState {
+    Uninitialized,
+    Initialized,
+    Frozen,
+}
+
+impl From<AccountState> for UiAccountState {
+    fn from(state: AccountState) -> Self {
+        match state {
+            AccountState::Uninitialized => UiAccountState::Uninitialized,
+            AccountState::Initialized => UiAccountState::Initialized,
+            AccountState::Frozen => UiAccountState::Frozen,
+        }
+    }
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -134,9 +169,11 @@ pub fn token_amount_to_ui_amount(amount: u64, decimals: u8) -> UiTokenAmount {
 #[derive(Debug, Serialize, Deserialize, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct UiMint {
-    pub owner: Option<String>,
+    pub mint_authority: Option<String>,
+    pub supply: StringAmount,
     pub decimals: u8,
     pub is_initialized: bool,
+    pub freeze_authority: Option<String>,
 }
 
 #[derive(Debug, Serialize, Deserialize, PartialEq)]
@@ -149,7 +186,7 @@ pub struct UiMultisig {
 }
 
 pub fn get_token_account_mint(data: &[u8]) -> Option<Pubkey> {
-    if data.len() == size_of::<Account>() {
+    if data.len() == Account::get_packed_len() {
         Some(Pubkey::new(&data[0..32]))
     } else {
         None
@@ -159,18 +196,23 @@ pub fn get_token_account_mint(data: &[u8]) -> Option<Pubkey> {
 #[cfg(test)]
 mod test {
     use super::*;
-    use spl_token_v1_0::state::unpack_unchecked;
 
     #[test]
     fn test_parse_token() {
         let mint_pubkey = SplTokenPubkey::new(&[2; 32]);
         let owner_pubkey = SplTokenPubkey::new(&[3; 32]);
-        let mut account_data = [0; size_of::<Account>()];
-        let mut account: &mut Account = unpack_unchecked(&mut account_data).unwrap();
-        account.mint = mint_pubkey;
-        account.owner = owner_pubkey;
-        account.amount = 42;
-        account.is_initialized = true;
+        let mut account_data = vec![0; Account::get_packed_len()];
+        Account::unpack_unchecked_mut(&mut account_data, &mut |account: &mut Account| {
+            account.mint = mint_pubkey;
+            account.owner = owner_pubkey;
+            account.amount = 42;
+            account.state = AccountState::Initialized;
+            account.is_native = COption::None;
+            account.close_authority = COption::Some(owner_pubkey);
+            Ok(())
+        })
+        .unwrap();
+
         assert!(parse_token(&account_data, None).is_err());
         assert_eq!(
             parse_token(&account_data, Some(2)).unwrap(),
@@ -183,39 +225,52 @@ mod test {
                     amount: "42".to_string()
                 },
                 delegate: None,
-                is_initialized: true,
+                state: UiAccountState::Initialized,
                 is_native: false,
+                rent_exempt_reserve: None,
                 delegated_amount: None,
+                close_authority: Some(owner_pubkey.to_string()),
             }),
         );
 
-        let mut mint_data = [0; size_of::<Mint>()];
-        let mut mint: &mut Mint = unpack_unchecked(&mut mint_data).unwrap();
-        mint.owner = COption::Some(owner_pubkey);
-        mint.decimals = 3;
-        mint.is_initialized = true;
+        let mut mint_data = vec![0; Mint::get_packed_len()];
+        Mint::unpack_unchecked_mut(&mut mint_data, &mut |mint: &mut Mint| {
+            mint.mint_authority = COption::Some(owner_pubkey);
+            mint.supply = 42;
+            mint.decimals = 3;
+            mint.is_initialized = true;
+            mint.freeze_authority = COption::Some(owner_pubkey);
+            Ok(())
+        })
+        .unwrap();
+
         assert_eq!(
             parse_token(&mint_data, None).unwrap(),
             TokenAccountType::Mint(UiMint {
-                owner: Some(owner_pubkey.to_string()),
+                mint_authority: Some(owner_pubkey.to_string()),
+                supply: 42.to_string(),
                 decimals: 3,
                 is_initialized: true,
+                freeze_authority: Some(owner_pubkey.to_string()),
             }),
         );
 
         let signer1 = SplTokenPubkey::new(&[1; 32]);
         let signer2 = SplTokenPubkey::new(&[2; 32]);
         let signer3 = SplTokenPubkey::new(&[3; 32]);
-        let mut multisig_data = [0; size_of::<Multisig>()];
-        let mut multisig: &mut Multisig = unpack_unchecked(&mut multisig_data).unwrap();
+        let mut multisig_data = vec![0; Multisig::get_packed_len()];
         let mut signers = [SplTokenPubkey::default(); 11];
         signers[0] = signer1;
         signers[1] = signer2;
         signers[2] = signer3;
-        multisig.m = 2;
-        multisig.n = 3;
-        multisig.is_initialized = true;
-        multisig.signers = signers;
+        Multisig::unpack_unchecked_mut(&mut multisig_data, &mut |multisig: &mut Multisig| {
+            multisig.m = 2;
+            multisig.n = 3;
+            multisig.is_initialized = true;
+            multisig.signers = signers;
+            Ok(())
+        })
+        .unwrap();
         assert_eq!(
             parse_token(&multisig_data, None).unwrap(),
             TokenAccountType::Multisig(UiMultisig {
@@ -237,9 +292,12 @@ mod test {
     #[test]
     fn test_get_token_account_mint() {
         let mint_pubkey = SplTokenPubkey::new(&[2; 32]);
-        let mut account_data = [0; size_of::<Account>()];
-        let mut account: &mut Account = unpack_unchecked(&mut account_data).unwrap();
-        account.mint = mint_pubkey;
+        let mut account_data = vec![0; Account::get_packed_len()];
+        Account::unpack_unchecked_mut(&mut account_data, &mut |account: &mut Account| {
+            account.mint = mint_pubkey;
+            Ok(())
+        })
+        .unwrap();
 
         let expected_mint_pubkey = Pubkey::new(&[2; 32]);
         assert_eq!(

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,10 +10,10 @@ homepage = "https://solana.com/"
 [dependencies]
 log = "0.4.6"
 rayon = "1.3.1"
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-runtime = { path = "../runtime", version = "1.3.4" }
-solana-measure = { path = "../measure", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-runtime = { path = "../runtime", version = "1.3.6" }
+solana-measure = { path = "../measure", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,16 +13,16 @@ crossbeam-channel = "0.4"
 log = "0.4.6"
 rand = "0.7.0"
 rayon = "1.3.1"
-solana-core = { path = "../core", version = "1.3.4" }
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-streamer = { path = "../streamer", version = "1.3.4" }
-solana-perf = { path = "../perf", version = "1.3.4" }
-solana-ledger = { path = "../ledger", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-runtime = { path = "../runtime", version = "1.3.4" }
-solana-measure = { path = "../measure", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
+solana-core = { path = "../core", version = "1.3.6" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-streamer = { path = "../streamer", version = "1.3.6" }
+solana-perf = { path = "../perf", version = "1.3.6" }
+solana-ledger = { path = "../ledger", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-runtime = { path = "../runtime", version = "1.3.6" }
+solana-measure = { path = "../measure", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banks-client/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "solana-banks-client"
+version = "1.3.6"
+description = "Solana banks client"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+async-trait = "0.1.36"
+bincode = "1.3.1"
+futures = "0.3"
+solana-banks-interface = { path = "../banks-interface", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+tarpc = { version = "0.21.0", features = ["full"] }
+tokio = "0.2"
+tokio-serde = { version = "0.6", features = ["bincode"] }
+
+[dev-dependencies]
+solana-runtime = { path = "../runtime", version = "1.3.6" }
+solana-banks-server = { path = "../banks-server", version = "1.3.6" }
+
+[lib]
+crate-type = ["lib"]
+name = "solana_banks_client"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

banks-client/src/lib.rs

@@ -0,0 +1,283 @@
+//! A client for the ledger state, from the perspective of an arbitrary validator.
+//!
+//! Use start_tcp_client() to create a client and then import BanksClientExt to
+//! access its methods. Additional "*_with_context" methods are also available,
+//! but they are undocumented, may change over time, and are generally more
+//! cumbersome to use.
+
+use async_trait::async_trait;
+use futures::future::join_all;
+pub use solana_banks_interface::{BanksClient, TransactionStatus};
+use solana_banks_interface::{BanksRequest, BanksResponse};
+use solana_sdk::{
+    account::Account, clock::Slot, commitment_config::CommitmentLevel,
+    fee_calculator::FeeCalculator, hash::Hash, pubkey::Pubkey, signature::Signature,
+    transaction::Transaction, transport,
+};
+use std::io::{self, Error, ErrorKind};
+use tarpc::{
+    client, context,
+    rpc::{transport::channel::UnboundedChannel, ClientMessage, Response},
+    serde_transport::tcp,
+};
+use tokio::{net::ToSocketAddrs, time::Duration};
+use tokio_serde::formats::Bincode;
+
+#[async_trait]
+pub trait BanksClientExt {
+    /// Send a transaction and return immediately. The server will resend the
+    /// transaction until either it is accepted by the cluster or the transaction's
+    /// blockhash expires.
+    async fn send_transaction(&mut self, transaction: Transaction) -> io::Result<()>;
+
+    /// Return a recent, rooted blockhash from the server. The cluster will only accept
+    /// transactions with a blockhash that has not yet expired. Use the `get_fees`
+    /// method to get both a blockhash and the blockhash's last valid slot.
+    async fn get_recent_blockhash(&mut self) -> io::Result<Hash>;
+
+    /// Return the fee parameters associated with a recent, rooted blockhash. The cluster
+    /// will use the transaction's blockhash to look up these same fee parameters and
+    /// use them to calculate the transaction fee.
+    async fn get_fees(&mut self) -> io::Result<(FeeCalculator, Hash, Slot)>;
+
+    /// Send a transaction and return after the transaction has been rejected or
+    /// reached the given level of commitment.
+    async fn process_transaction_with_commitment(
+        &mut self,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> transport::Result<()>;
+
+    /// Send a transaction and return after the transaction has been finalized or rejected.
+    async fn process_transaction(&mut self, transaction: Transaction) -> transport::Result<()>;
+
+    /// Return the status of a transaction with a signature matching the transaction's first
+    /// signature. Return None if the transaction is not found, which may be because the
+    /// blockhash was expired or the fee-paying account had insufficient funds to pay the
+    /// transaction fee. Note that servers rarely store the full transaction history. This
+    /// method may return None if the transaction status has been discarded.
+    async fn get_transaction_status(
+        &mut self,
+        signature: Signature,
+    ) -> io::Result<Option<TransactionStatus>>;
+
+    /// Same as get_transaction_status, but for multiple transactions.
+    async fn get_transaction_statuses(
+        &mut self,
+        signatures: Vec<Signature>,
+    ) -> io::Result<Vec<Option<TransactionStatus>>>;
+
+    /// Return the most recent rooted slot height. All transactions at or below this height
+    /// are said to be finalized. The cluster will not fork to a higher slot height.
+    async fn get_root_slot(&mut self) -> io::Result<Slot>;
+
+    /// Return the account at the given address at the time of the most recent root slot.
+    /// If the account is not found, None is returned.
+    async fn get_account(&mut self, address: Pubkey) -> io::Result<Option<Account>>;
+
+    /// Return the balance in lamports of an account at the given address at the slot
+    /// corresponding to the given commitment level.
+    async fn get_balance_with_commitment(
+        &mut self,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> io::Result<u64>;
+
+    /// Return the balance in lamports of an account at the given address at the time
+    /// of the most recent root slot.
+    async fn get_balance(&mut self, address: Pubkey) -> io::Result<u64>;
+}
+
+#[async_trait]
+impl BanksClientExt for BanksClient {
+    async fn send_transaction(&mut self, transaction: Transaction) -> io::Result<()> {
+        self.send_transaction_with_context(context::current(), transaction)
+            .await
+    }
+
+    async fn get_fees(&mut self) -> io::Result<(FeeCalculator, Hash, Slot)> {
+        self.get_fees_with_commitment_and_context(context::current(), CommitmentLevel::Root)
+            .await
+    }
+
+    async fn get_recent_blockhash(&mut self) -> io::Result<Hash> {
+        Ok(self.get_fees().await?.1)
+    }
+
+    async fn process_transaction_with_commitment(
+        &mut self,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> transport::Result<()> {
+        let mut ctx = context::current();
+        ctx.deadline += Duration::from_secs(50);
+        let result = self
+            .process_transaction_with_commitment_and_context(ctx, transaction, commitment)
+            .await?;
+        match result {
+            None => Err(Error::new(ErrorKind::TimedOut, "invalid blockhash or fee-payer").into()),
+            Some(transaction_result) => Ok(transaction_result?),
+        }
+    }
+
+    async fn process_transaction(&mut self, transaction: Transaction) -> transport::Result<()> {
+        self.process_transaction_with_commitment(transaction, CommitmentLevel::default())
+            .await
+    }
+
+    async fn get_root_slot(&mut self) -> io::Result<Slot> {
+        self.get_slot_with_context(context::current(), CommitmentLevel::Root)
+            .await
+    }
+
+    async fn get_account(&mut self, address: Pubkey) -> io::Result<Option<Account>> {
+        self.get_account_with_commitment_and_context(
+            context::current(),
+            address,
+            CommitmentLevel::default(),
+        )
+        .await
+    }
+
+    async fn get_balance_with_commitment(
+        &mut self,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> io::Result<u64> {
+        let account = self
+            .get_account_with_commitment_and_context(context::current(), address, commitment)
+            .await?;
+        Ok(account.map(|x| x.lamports).unwrap_or(0))
+    }
+
+    async fn get_balance(&mut self, address: Pubkey) -> io::Result<u64> {
+        self.get_balance_with_commitment(address, CommitmentLevel::default())
+            .await
+    }
+
+    async fn get_transaction_status(
+        &mut self,
+        signature: Signature,
+    ) -> io::Result<Option<TransactionStatus>> {
+        self.get_transaction_status_with_context(context::current(), signature)
+            .await
+    }
+
+    async fn get_transaction_statuses(
+        &mut self,
+        signatures: Vec<Signature>,
+    ) -> io::Result<Vec<Option<TransactionStatus>>> {
+        // tarpc futures oddly hold a mutable reference back to the client so clone the client upfront
+        let mut clients_and_signatures: Vec<_> = signatures
+            .into_iter()
+            .map(|signature| (self.clone(), signature))
+            .collect();
+
+        let futs = clients_and_signatures
+            .iter_mut()
+            .map(|(client, signature)| client.get_transaction_status(*signature));
+
+        let statuses = join_all(futs).await;
+
+        // Convert Vec<Result<_, _>> to Result<Vec<_>>
+        statuses.into_iter().collect()
+    }
+}
+
+pub async fn start_client(
+    transport: UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>>,
+) -> io::Result<BanksClient> {
+    BanksClient::new(client::Config::default(), transport).spawn()
+}
+
+pub async fn start_tcp_client<T: ToSocketAddrs>(addr: T) -> io::Result<BanksClient> {
+    let transport = tcp::connect(addr, Bincode::default()).await?;
+    BanksClient::new(client::Config::default(), transport).spawn()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_banks_server::banks_server::start_local_server;
+    use solana_runtime::{bank::Bank, bank_forks::BankForks, genesis_utils::create_genesis_config};
+    use solana_sdk::{message::Message, pubkey::Pubkey, signature::Signer, system_instruction};
+    use std::sync::{Arc, RwLock};
+    use tarpc::transport;
+    use tokio::{runtime::Runtime, time::delay_for};
+
+    #[test]
+    fn test_banks_client_new() {
+        let (client_transport, _server_transport) = transport::channel::unbounded();
+        BanksClient::new(client::Config::default(), client_transport);
+    }
+
+    #[test]
+    fn test_banks_server_transfer_via_server() -> io::Result<()> {
+        // This test shows the preferred way to interact with BanksServer.
+        // It creates a runtime explicitly (no globals via tokio macros) and calls
+        // `runtime.block_on()` just once, to run all the async code.
+
+        let genesis = create_genesis_config(10);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new(
+            &genesis.genesis_config,
+        ))));
+
+        let bob_pubkey = Pubkey::new_rand();
+        let mint_pubkey = genesis.mint_keypair.pubkey();
+        let instruction = system_instruction::transfer(&mint_pubkey, &bob_pubkey, 1);
+        let message = Message::new(&[instruction], Some(&mint_pubkey));
+
+        Runtime::new()?.block_on(async {
+            let client_transport = start_local_server(&bank_forks).await;
+            let mut banks_client =
+                BanksClient::new(client::Config::default(), client_transport).spawn()?;
+
+            let recent_blockhash = banks_client.get_recent_blockhash().await?;
+            let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);
+            banks_client.process_transaction(transaction).await.unwrap();
+            assert_eq!(banks_client.get_balance(bob_pubkey).await?, 1);
+            Ok(())
+        })
+    }
+
+    #[test]
+    fn test_banks_server_transfer_via_client() -> io::Result<()> {
+        // The caller may not want to hold the connection open until the transaction
+        // is processed (or blockhash expires). In this test, we verify the
+        // server-side functionality is available to the client.
+
+        let genesis = create_genesis_config(10);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new(
+            &genesis.genesis_config,
+        ))));
+
+        let mint_pubkey = &genesis.mint_keypair.pubkey();
+        let bob_pubkey = Pubkey::new_rand();
+        let instruction = system_instruction::transfer(&mint_pubkey, &bob_pubkey, 1);
+        let message = Message::new(&[instruction], Some(&mint_pubkey));
+
+        Runtime::new()?.block_on(async {
+            let client_transport = start_local_server(&bank_forks).await;
+            let mut banks_client =
+                BanksClient::new(client::Config::default(), client_transport).spawn()?;
+            let (_, recent_blockhash, last_valid_slot) = banks_client.get_fees().await?;
+            let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);
+            let signature = transaction.signatures[0];
+            banks_client.send_transaction(transaction).await?;
+
+            let mut status = banks_client.get_transaction_status(signature).await?;
+
+            while status.is_none() {
+                let root_slot = banks_client.get_root_slot().await?;
+                if root_slot > last_valid_slot {
+                    break;
+                }
+                delay_for(Duration::from_millis(100)).await;
+                status = banks_client.get_transaction_status(signature).await?;
+            }
+            assert!(status.unwrap().err.is_none());
+            assert_eq!(banks_client.get_balance(bob_pubkey).await?, 1);
+            Ok(())
+        })
+    }
+}

banks-interface/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "solana-banks-interface"
+version = "1.3.6"
+description = "Solana banks RPC interface"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+serde = { version = "1.0.112", features = ["derive"] }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+tarpc = { version = "0.21.0", features = ["full"] }
+
+[lib]
+crate-type = ["lib"]
+name = "solana_banks_interface"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

banks-interface/src/lib.rs

@@ -0,0 +1,49 @@
+use serde::{Deserialize, Serialize};
+use solana_sdk::{
+    account::Account,
+    clock::Slot,
+    commitment_config::CommitmentLevel,
+    fee_calculator::FeeCalculator,
+    hash::Hash,
+    pubkey::Pubkey,
+    signature::Signature,
+    transaction::{self, Transaction, TransactionError},
+};
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct TransactionStatus {
+    pub slot: Slot,
+    pub confirmations: Option<usize>, // None = rooted
+    pub err: Option<TransactionError>,
+}
+
+#[tarpc::service]
+pub trait Banks {
+    async fn send_transaction_with_context(transaction: Transaction);
+    async fn get_fees_with_commitment_and_context(
+        commitment: CommitmentLevel,
+    ) -> (FeeCalculator, Hash, Slot);
+    async fn get_transaction_status_with_context(signature: Signature)
+        -> Option<TransactionStatus>;
+    async fn get_slot_with_context(commitment: CommitmentLevel) -> Slot;
+    async fn process_transaction_with_commitment_and_context(
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> Option<transaction::Result<()>>;
+    async fn get_account_with_commitment_and_context(
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> Option<Account>;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tarpc::{client, transport};
+
+    #[test]
+    fn test_banks_client_new() {
+        let (client_transport, _server_transport) = transport::channel::unbounded();
+        BanksClient::new(client::Config::default(), client_transport);
+    }
+}

banks-server/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "solana-banks-server"
+version = "1.3.6"
+description = "Solana banks server"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+bincode = "1.3.1"
+futures = "0.3"
+solana-banks-interface = { path = "../banks-interface", version = "1.3.6" }
+solana-runtime = { path = "../runtime", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+tarpc = { version = "0.21.0", features = ["full"] }
+tokio = "0.2"
+tokio-serde = { version = "0.6", features = ["bincode"] }
+
+[lib]
+crate-type = ["lib"]
+name = "solana_banks_server"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

banks-server/src/banks_server.rs

@@ -0,0 +1,272 @@
+use bincode::{deserialize, serialize};
+use futures::{
+    future,
+    prelude::stream::{self, StreamExt},
+};
+use solana_banks_interface::{Banks, BanksRequest, BanksResponse, TransactionStatus};
+use solana_runtime::{
+    bank::Bank,
+    bank_forks::BankForks,
+    commitment::{BlockCommitmentCache, CommitmentSlots},
+    send_transaction_service::{SendTransactionService, TransactionInfo},
+};
+use solana_sdk::{
+    account::Account,
+    clock::Slot,
+    commitment_config::CommitmentLevel,
+    fee_calculator::FeeCalculator,
+    hash::Hash,
+    pubkey::Pubkey,
+    signature::Signature,
+    transaction::{self, Transaction},
+};
+use std::{
+    collections::HashMap,
+    io,
+    net::SocketAddr,
+    sync::{
+        atomic::AtomicBool,
+        mpsc::{channel, Receiver, Sender},
+        Arc, RwLock,
+    },
+    thread::Builder,
+    time::Duration,
+};
+use tarpc::{
+    context::Context,
+    rpc::{transport::channel::UnboundedChannel, ClientMessage, Response},
+    serde_transport::tcp,
+    server::{self, Channel, Handler},
+    transport,
+};
+use tokio::time::delay_for;
+use tokio_serde::formats::Bincode;
+
+#[derive(Clone)]
+struct BanksServer {
+    bank_forks: Arc<RwLock<BankForks>>,
+    block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+    transaction_sender: Sender<TransactionInfo>,
+}
+
+impl BanksServer {
+    /// Return a BanksServer that forwards transactions to the
+    /// given sender. If unit-testing, those transactions can go to
+    /// a bank in the given BankForks. Otherwise, the receiver should
+    /// forward them to a validator in the leader schedule.
+    fn new(
+        bank_forks: Arc<RwLock<BankForks>>,
+        block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+        transaction_sender: Sender<TransactionInfo>,
+    ) -> Self {
+        Self {
+            bank_forks,
+            block_commitment_cache,
+            transaction_sender,
+        }
+    }
+
+    fn run(bank: &Bank, transaction_receiver: Receiver<TransactionInfo>) {
+        while let Ok(info) = transaction_receiver.recv() {
+            let mut transaction_infos = vec![info];
+            while let Ok(info) = transaction_receiver.try_recv() {
+                transaction_infos.push(info);
+            }
+            let transactions: Vec<_> = transaction_infos
+                .into_iter()
+                .map(|info| deserialize(&info.wire_transaction).unwrap())
+                .collect();
+            let _ = bank.process_transactions(&transactions);
+        }
+    }
+
+    /// Useful for unit-testing
+    fn new_loopback(bank_forks: Arc<RwLock<BankForks>>) -> Self {
+        let (transaction_sender, transaction_receiver) = channel();
+        let bank = bank_forks.read().unwrap().working_bank();
+        let slot = bank.slot();
+        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new(
+            HashMap::default(),
+            0,
+            CommitmentSlots {
+                slot,
+                root: 0,
+                highest_confirmed_slot: 0,
+                highest_confirmed_root: 0,
+            },
+        )));
+        Builder::new()
+            .name("solana-bank-forks-client".to_string())
+            .spawn(move || Self::run(&bank, transaction_receiver))
+            .unwrap();
+        Self::new(bank_forks, block_commitment_cache, transaction_sender)
+    }
+
+    fn slot(&self, commitment: CommitmentLevel) -> Slot {
+        self.block_commitment_cache
+            .read()
+            .unwrap()
+            .slot_with_commitment(commitment)
+    }
+
+    fn bank(&self, commitment: CommitmentLevel) -> Arc<Bank> {
+        self.bank_forks.read().unwrap()[self.slot(commitment)].clone()
+    }
+
+    async fn poll_signature_status(
+        self,
+        signature: Signature,
+        last_valid_slot: Slot,
+        commitment: CommitmentLevel,
+    ) -> Option<transaction::Result<()>> {
+        let mut status = self.bank(commitment).get_signature_status(&signature);
+        while status.is_none() {
+            delay_for(Duration::from_millis(200)).await;
+            let bank = self.bank(commitment);
+            if bank.slot() > last_valid_slot {
+                break;
+            }
+            status = bank.get_signature_status(&signature);
+        }
+        status
+    }
+}
+
+#[tarpc::server]
+impl Banks for BanksServer {
+    async fn send_transaction_with_context(self, _: Context, transaction: Transaction) {
+        let blockhash = &transaction.message.recent_blockhash;
+        let last_valid_slot = self
+            .bank_forks
+            .read()
+            .unwrap()
+            .root_bank()
+            .get_blockhash_last_valid_slot(&blockhash)
+            .unwrap();
+        let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
+        let info =
+            TransactionInfo::new(signature, serialize(&transaction).unwrap(), last_valid_slot);
+        self.transaction_sender.send(info).unwrap();
+    }
+
+    async fn get_fees_with_commitment_and_context(
+        self,
+        _: Context,
+        commitment: CommitmentLevel,
+    ) -> (FeeCalculator, Hash, Slot) {
+        let bank = self.bank(commitment);
+        let (blockhash, fee_calculator) = bank.last_blockhash_with_fee_calculator();
+        let last_valid_slot = bank.get_blockhash_last_valid_slot(&blockhash).unwrap();
+        (fee_calculator, blockhash, last_valid_slot)
+    }
+
+    async fn get_transaction_status_with_context(
+        self,
+        _: Context,
+        signature: Signature,
+    ) -> Option<TransactionStatus> {
+        let bank = self.bank(CommitmentLevel::Recent);
+        let (slot, status) = bank.get_signature_status_slot(&signature)?;
+        let r_block_commitment_cache = self.block_commitment_cache.read().unwrap();
+
+        let confirmations = if r_block_commitment_cache.root() >= slot {
+            None
+        } else {
+            r_block_commitment_cache
+                .get_confirmation_count(slot)
+                .or(Some(0))
+        };
+        Some(TransactionStatus {
+            slot,
+            confirmations,
+            err: status.err(),
+        })
+    }
+
+    async fn get_slot_with_context(self, _: Context, commitment: CommitmentLevel) -> Slot {
+        self.slot(commitment)
+    }
+
+    async fn process_transaction_with_commitment_and_context(
+        self,
+        _: Context,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> Option<transaction::Result<()>> {
+        let blockhash = &transaction.message.recent_blockhash;
+        let last_valid_slot = self
+            .bank_forks
+            .read()
+            .unwrap()
+            .root_bank()
+            .get_blockhash_last_valid_slot(&blockhash)
+            .unwrap();
+        let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
+        let info =
+            TransactionInfo::new(signature, serialize(&transaction).unwrap(), last_valid_slot);
+        self.transaction_sender.send(info).unwrap();
+        self.poll_signature_status(signature, last_valid_slot, commitment)
+            .await
+    }
+
+    async fn get_account_with_commitment_and_context(
+        self,
+        _: Context,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> Option<Account> {
+        let bank = self.bank(commitment);
+        bank.get_account(&address)
+    }
+}
+
+pub async fn start_local_server(
+    bank_forks: &Arc<RwLock<BankForks>>,
+) -> UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>> {
+    let banks_server = BanksServer::new_loopback(bank_forks.clone());
+    let (client_transport, server_transport) = transport::channel::unbounded();
+    let server = server::new(server::Config::default())
+        .incoming(stream::once(future::ready(server_transport)))
+        .respond_with(banks_server.serve());
+    tokio::spawn(server);
+    client_transport
+}
+
+pub async fn start_tcp_server(
+    listen_addr: SocketAddr,
+    tpu_addr: SocketAddr,
+    bank_forks: Arc<RwLock<BankForks>>,
+    block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+) -> io::Result<()> {
+    // Note: These settings are copied straight from the tarpc example.
+    let server = tcp::listen(listen_addr, Bincode::default)
+        .await?
+        // Ignore accept errors.
+        .filter_map(|r| future::ready(r.ok()))
+        .map(server::BaseChannel::with_defaults)
+        // Limit channels to 1 per IP.
+        .max_channels_per_key(1, |t| t.as_ref().peer_addr().unwrap().ip())
+        // serve is generated by the service attribute. It takes as input any type implementing
+        // the generated Banks trait.
+        .map(move |chan| {
+            let (sender, receiver) = channel();
+            let exit_send_transaction_service = Arc::new(AtomicBool::new(false));
+
+            SendTransactionService::new(
+                tpu_addr,
+                &bank_forks,
+                &exit_send_transaction_service,
+                receiver,
+            );
+
+            let server =
+                BanksServer::new(bank_forks.clone(), block_commitment_cache.clone(), sender);
+            chan.respond_with(server.serve()).execute()
+        })
+        // Max 10 channels.
+        .buffer_unordered(10)
+        .for_each(|_| async {});
+
+    server.await;
+    Ok(())
+}

banks-server/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod banks_server;
+pub mod rpc_banks_service;

banks-server/src/rpc_banks_service.rs

@@ -0,0 +1,116 @@
+//! The `rpc_banks_service` module implements the Solana Banks RPC API.
+
+use crate::banks_server::start_tcp_server;
+use futures::{future::FutureExt, pin_mut, prelude::stream::StreamExt, select};
+use solana_runtime::{bank_forks::BankForks, commitment::BlockCommitmentCache};
+use std::{
+    net::SocketAddr,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc, RwLock,
+    },
+    thread::{self, Builder, JoinHandle},
+};
+use tokio::{
+    runtime::Runtime,
+    time::{self, Duration},
+};
+
+pub struct RpcBanksService {
+    thread_hdl: JoinHandle<()>,
+}
+
+/// Run the TCP service until `exit` is set to true
+async fn start_abortable_tcp_server(
+    listen_addr: SocketAddr,
+    tpu_addr: SocketAddr,
+    bank_forks: Arc<RwLock<BankForks>>,
+    block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+    exit: Arc<AtomicBool>,
+) {
+    let server = start_tcp_server(
+        listen_addr,
+        tpu_addr,
+        bank_forks.clone(),
+        block_commitment_cache.clone(),
+    )
+    .fuse();
+    let interval = time::interval(Duration::from_millis(100)).fuse();
+    pin_mut!(server, interval);
+    loop {
+        select! {
+            _ = server => {},
+            _ = interval.select_next_some() => {
+                if exit.load(Ordering::Relaxed) {
+                    break;
+                }
+            }
+        }
+    }
+}
+
+impl RpcBanksService {
+    fn run(
+        listen_addr: SocketAddr,
+        tpu_addr: SocketAddr,
+        bank_forks: Arc<RwLock<BankForks>>,
+        block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+        exit: Arc<AtomicBool>,
+    ) {
+        let server = start_abortable_tcp_server(
+            listen_addr,
+            tpu_addr,
+            bank_forks,
+            block_commitment_cache,
+            exit,
+        );
+        Runtime::new().unwrap().block_on(server);
+    }
+
+    pub fn new(
+        listen_addr: SocketAddr,
+        tpu_addr: SocketAddr,
+        bank_forks: &Arc<RwLock<BankForks>>,
+        block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
+        exit: &Arc<AtomicBool>,
+    ) -> Self {
+        let bank_forks = bank_forks.clone();
+        let block_commitment_cache = block_commitment_cache.clone();
+        let exit = exit.clone();
+        let thread_hdl = Builder::new()
+            .name("solana-rpc-banks".to_string())
+            .spawn(move || {
+                Self::run(
+                    listen_addr,
+                    tpu_addr,
+                    bank_forks,
+                    block_commitment_cache,
+                    exit,
+                )
+            })
+            .unwrap();
+
+        Self { thread_hdl }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.thread_hdl.join()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_runtime::bank::Bank;
+
+    #[test]
+    fn test_rpc_banks_server_exit() {
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::default())));
+        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
+        let exit = Arc::new(AtomicBool::new(false));
+        let addr = "127.0.0.1:0".parse().unwrap();
+        let service = RpcBanksService::new(addr, addr, &bank_forks, &block_commitment_cache, &exit);
+        exit.store(true, Ordering::Relaxed);
+        service.join().unwrap();
+    }
+}

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,21 +18,21 @@ rand = "0.7.0"
 rayon = "1.3.1"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-core = { path = "../core", version = "1.3.4" }
-solana-genesis = { path = "../genesis", version = "1.3.4" }
-solana-client = { path = "../client", version = "1.3.4" }
-solana-faucet = { path = "../faucet", version = "1.3.4" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-metrics = { path = "../metrics", version = "1.3.4" }
-solana-net-utils = { path = "../net-utils", version = "1.3.4" }
-solana-runtime = { path = "../runtime", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-core = { path = "../core", version = "1.3.6" }
+solana-genesis = { path = "../genesis", version = "1.3.6" }
+solana-client = { path = "../client", version = "1.3.6" }
+solana-faucet = { path = "../faucet", version = "1.3.6" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-metrics = { path = "../metrics", version = "1.3.6" }
+solana-net-utils = { path = "../net-utils", version = "1.3.6" }
+solana-runtime = { path = "../runtime", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.3.4" }
+solana-local-cluster = { path = "../local-cluster", version = "1.3.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/Cargo.toml

@@ -2,18 +2,18 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-streamer = { path = "../streamer", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-net-utils = { path = "../net-utils", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-streamer = { path = "../streamer", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-net-utils = { path = "../net-utils", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,23 +14,23 @@ log = "0.4.8"
 rayon = "1.3.1"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-core = { path = "../core", version = "1.3.4" }
-solana-genesis = { path = "../genesis", version = "1.3.4" }
-solana-client = { path = "../client", version = "1.3.4" }
-solana-faucet = { path = "../faucet", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-metrics = { path = "../metrics", version = "1.3.4" }
-solana-measure = { path = "../measure", version = "1.3.4" }
-solana-net-utils = { path = "../net-utils", version = "1.3.4" }
-solana-runtime = { path = "../runtime", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-core = { path = "../core", version = "1.3.6" }
+solana-genesis = { path = "../genesis", version = "1.3.6" }
+solana-client = { path = "../client", version = "1.3.6" }
+solana-faucet = { path = "../faucet", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-metrics = { path = "../metrics", version = "1.3.6" }
+solana-measure = { path = "../measure", version = "1.3.6" }
+solana-net-utils = { path = "../net-utils", version = "1.3.6" }
+solana-runtime = { path = "../runtime", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.3.4" }
+solana-local-cluster = { path = "../local-cluster", version = "1.3.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

ci/channel-info.sh

@@ -89,11 +89,17 @@ BETA_CHANNEL_LATEST_TAG=${beta_tag:+v$beta_tag}
 STABLE_CHANNEL_LATEST_TAG=${stable_tag:+v$stable_tag}
 
 
-if [[ $CI_BRANCH = "$STABLE_CHANNEL" ]]; then
+if [[ -n $CI_BASE_BRANCH ]]; then
+  BRANCH="$CI_BASE_BRANCH"
+elif [[ -n $CI_BRANCH ]]; then
+  BRANCH="$CI_BRANCH"
+fi
+
+if [[ $BRANCH = "$STABLE_CHANNEL" ]]; then
   CHANNEL=stable
-elif [[ $CI_BRANCH = "$EDGE_CHANNEL" ]]; then
+elif [[ $BRANCH = "$EDGE_CHANNEL" ]]; then
   CHANNEL=edge
-elif [[ $CI_BRANCH = "$BETA_CHANNEL" ]]; then
+elif [[ $BRANCH = "$BETA_CHANNEL" ]]; then
   CHANNEL=beta
 fi
 

ci/setup-new-buildkite-agent/setup-buildkite.sh

@@ -76,7 +76,7 @@ RestartForceExitStatus=SIGPIPE
 TimeoutStartSec=10
 TimeoutStopSec=0
 KillMode=process
-LimitNOFILE=65536
+LimitNOFILE=500000
 
 [Install]
 WantedBy=multi-user.target

ci/setup-new-buildkite-agent/setup-limits.sh

@@ -8,5 +8,5 @@ source "$HERE"/utils.sh
 ensure_env || exit 1
 
 # Allow more files to be opened by a user
-sed -i 's/^\(# End of file\)/* soft nofile 65535\n\n\1/' /etc/security/limits.conf
+echo "* - nofile 500000" > /etc/security/limits.d/90-solana-nofiles.conf
 

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
 thiserror = "1.0.20"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

cli-config/src/config.rs

@@ -76,6 +76,17 @@ impl Config {
         ws_url.to_string()
     }
 
+    pub fn compute_rpc_banks_url(json_rpc_url: &str) -> String {
+        let json_rpc_url: Option<Url> = json_rpc_url.parse().ok();
+        if json_rpc_url.is_none() {
+            return "".to_string();
+        }
+        let mut url = json_rpc_url.unwrap();
+        let port = url.port_or_known_default().unwrap_or(80);
+        url.set_port(Some(port + 2)).expect("unable to set port");
+        url.to_string()
+    }
+
     pub fn import_address_labels<P>(&mut self, filename: P) -> Result<(), io::Error>
     where
         P: AsRef<Path>,
@@ -122,4 +133,28 @@ mod test {
 
         assert_eq!(Config::compute_websocket_url(&"garbage"), String::new());
     }
+
+    #[test]
+    fn compute_rpc_banks_url() {
+        assert_eq!(
+            Config::compute_rpc_banks_url(&"http://devnet.solana.com"),
+            "http://devnet.solana.com:82/".to_string()
+        );
+
+        assert_eq!(
+            Config::compute_rpc_banks_url(&"https://devnet.solana.com"),
+            "https://devnet.solana.com:445/".to_string()
+        );
+
+        assert_eq!(
+            Config::compute_rpc_banks_url(&"http://example.com:8899"),
+            "http://example.com:8901/".to_string()
+        );
+        assert_eq!(
+            Config::compute_rpc_banks_url(&"https://example.com:1234"),
+            "https://example.com:1236/".to_string()
+        );
+
+        assert_eq!(Config::compute_rpc_banks_url(&"garbage"), String::new());
+    }
 }

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -27,29 +27,29 @@ reqwest = { version = "0.10.6", default-features = false, features = ["blocking"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.3.4" }
-solana-budget-program = { path = "../programs/budget", version = "1.3.4" }
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-cli-config = { path = "../cli-config", version = "1.3.4" }
-solana-client = { path = "../client", version = "1.3.4" }
-solana-config-program = { path = "../programs/config", version = "1.3.4" }
-solana-faucet = { path = "../faucet", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-net-utils = { path = "../net-utils", version = "1.3.4" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.3.4" }
-solana-runtime = { path = "../runtime", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-stake-program = { path = "../programs/stake", version = "1.3.4" }
-solana-transaction-status = { path = "../transaction-status", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
-solana-vote-program = { path = "../programs/vote", version = "1.3.4" }
-solana-vote-signer = { path = "../vote-signer", version = "1.3.4" }
+solana-account-decoder = { path = "../account-decoder", version = "1.3.6" }
+solana-budget-program = { path = "../programs/budget", version = "1.3.6" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-cli-config = { path = "../cli-config", version = "1.3.6" }
+solana-client = { path = "../client", version = "1.3.6" }
+solana-config-program = { path = "../programs/config", version = "1.3.6" }
+solana-faucet = { path = "../faucet", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-net-utils = { path = "../net-utils", version = "1.3.6" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.3.6" }
+solana-runtime = { path = "../runtime", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-stake-program = { path = "../programs/stake", version = "1.3.6" }
+solana-transaction-status = { path = "../transaction-status", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
+solana-vote-program = { path = "../programs/vote", version = "1.3.6" }
+solana-vote-signer = { path = "../vote-signer", version = "1.3.6" }
 thiserror = "1.0.20"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.3.4" }
-solana-budget-program = { path = "../programs/budget", version = "1.3.4" }
+solana-core = { path = "../core", version = "1.3.6" }
+solana-budget-program = { path = "../programs/budget", version = "1.3.6" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cli.rs

@@ -33,7 +33,7 @@ use solana_faucet::faucet::request_airdrop_transaction;
 use solana_faucet::faucet_mock::request_airdrop_transaction;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
-    bpf_loader,
+    bpf_loader, bpf_loader_deprecated,
     clock::{Epoch, Slot, DEFAULT_TICKS_PER_SECOND},
     commitment_config::CommitmentConfig,
     decode_error::DecodeError,
@@ -266,6 +266,7 @@ pub enum CliCommand {
     Deploy {
         program_location: String,
         address: Option<SignerIndex>,
+        use_deprecated_loader: bool,
     },
     // Stake Commands
     CreateStakeAccount {
@@ -586,6 +587,7 @@ impl CliConfig<'_> {
         config.commitment = CommitmentConfig::recent();
         config.send_transaction_config = RpcSendTransactionConfig {
             skip_preflight: true,
+            ..RpcSendTransactionConfig::default()
         };
         config
     }
@@ -696,11 +698,13 @@ pub fn parse_command(
                 signers.push(signer);
                 1
             });
+            let use_deprecated_loader = matches.is_present("use_deprecated_loader");
 
             Ok(CliCommandInfo {
                 command: CliCommand::Deploy {
                     program_location: matches.value_of("program_location").unwrap().to_string(),
                     address,
+                    use_deprecated_loader,
                 },
                 signers,
             })
@@ -1295,6 +1299,7 @@ fn send_and_confirm_transactions_with_spinner<T: Signers>(
                     &transaction,
                     RpcSendTransactionConfig {
                         skip_preflight: true,
+                        ..RpcSendTransactionConfig::default()
                     },
                 )
                 .ok();
@@ -1364,6 +1369,7 @@ fn process_deploy(
     config: &CliConfig,
     program_location: &str,
     address: Option<SignerIndex>,
+    use_deprecated_loader: bool,
 ) -> ProcessResult {
     let new_keypair = Keypair::new(); // Create ephemeral keypair to use for program address, if not provided
     let program_id = if let Some(i) = address {
@@ -1379,6 +1385,12 @@ fn process_deploy(
         CliError::DynamicProgramError(format!("Unable to read program file: {}", err))
     })?;
 
+    let loader_id = if use_deprecated_loader {
+        bpf_loader_deprecated::id()
+    } else {
+        bpf_loader::id()
+    };
+
     // Build transactions to calculate fees
     let mut messages: Vec<&Message> = Vec::new();
     let (blockhash, fee_calculator, _) = rpc_client
@@ -1390,7 +1402,7 @@ fn process_deploy(
         &program_id.pubkey(),
         minimum_balance.max(1),
         program_data.len() as u64,
-        &bpf_loader::id(),
+        &loader_id,
     );
     let message = Message::new(&[ix], Some(&config.signers[0].pubkey()));
     let mut create_account_tx = Transaction::new_unsigned(message);
@@ -1401,7 +1413,7 @@ fn process_deploy(
     for (chunk, i) in program_data.chunks(DATA_CHUNK_SIZE).zip(0..) {
         let instruction = loader_instruction::write(
             &program_id.pubkey(),
-            &bpf_loader::id(),
+            &loader_id,
             (i * DATA_CHUNK_SIZE) as u32,
             chunk.to_vec(),
         );
@@ -1414,7 +1426,7 @@ fn process_deploy(
     }
     messages.append(&mut write_message_refs);
 
-    let instruction = loader_instruction::finalize(&program_id.pubkey(), &bpf_loader::id());
+    let instruction = loader_instruction::finalize(&program_id.pubkey(), &loader_id);
     let finalize_message = Message::new(&[instruction], Some(&signers[0].pubkey()));
     messages.push(&finalize_message);
 
@@ -1465,6 +1477,7 @@ fn process_deploy(
             config.commitment,
             RpcSendTransactionConfig {
                 skip_preflight: true,
+                ..RpcSendTransactionConfig::default()
             },
         )
         .map_err(|e| {
@@ -1964,7 +1977,14 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         CliCommand::Deploy {
             program_location,
             address,
-        } => process_deploy(&rpc_client, config, program_location, *address),
+            use_deprecated_loader,
+        } => process_deploy(
+            &rpc_client,
+            config,
+            program_location,
+            *address,
+            *use_deprecated_loader,
+        ),
 
         // Stake Commands
 
@@ -2651,6 +2671,13 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
                         .takes_value(true)
                         .validator(is_valid_signer)
                         .help("The signer for the desired address of the program [default: new random address]")
+                )
+                .arg(
+                    Arg::with_name("use-deprecated-loader")
+                        .long("use_deprecated_loader")
+                        .takes_value(false)
+                        .hidden(true) // Don't document this argument to discourage its use
+                        .help("Use the deprecated BPF loader")
                 ),
         )
         .subcommand(
@@ -3095,6 +3122,7 @@ mod tests {
                 command: CliCommand::Deploy {
                     program_location: "/Users/test/program.o".to_string(),
                     address: None,
+                    use_deprecated_loader: false,
                 },
                 signers: vec![read_keypair_file(&keypair_file).unwrap().into()],
             }
@@ -3115,6 +3143,7 @@ mod tests {
                 command: CliCommand::Deploy {
                     program_location: "/Users/test/program.o".to_string(),
                     address: Some(1),
+                    use_deprecated_loader: false,
                 },
                 signers: vec![
                     read_keypair_file(&keypair_file).unwrap().into(),
@@ -3834,6 +3863,7 @@ mod tests {
         config.command = CliCommand::Deploy {
             program_location: pathbuf.to_str().unwrap().to_string(),
             address: None,
+            use_deprecated_loader: false,
         };
         let result = process_command(&config);
         let json: Value = serde_json::from_str(&result.unwrap()).unwrap();
@@ -3851,6 +3881,7 @@ mod tests {
         config.command = CliCommand::Deploy {
             program_location: "bad/file/location.so".to_string(),
             address: None,
+            use_deprecated_loader: false,
         };
         assert!(process_command(&config).is_err());
     }

cli/tests/deploy.rs

@@ -63,6 +63,7 @@ fn test_cli_deploy_program() {
     config.command = CliCommand::Deploy {
         program_location: pathbuf.to_str().unwrap().to_string(),
         address: None,
+        use_deprecated_loader: false,
     };
 
     let response = process_command(&config);
@@ -96,6 +97,7 @@ fn test_cli_deploy_program() {
     config.command = CliCommand::Deploy {
         program_location: pathbuf.to_str().unwrap().to_string(),
         address: Some(1),
+        use_deprecated_loader: false,
     };
     process_command(&config).unwrap();
     let account1 = rpc_client

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,11 +19,11 @@ reqwest = { version = "0.10.6", default-features = false, features = ["blocking"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.3.4" }
-solana-net-utils = { path = "../net-utils", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-transaction-status = { path = "../transaction-status", version = "1.3.4" }
-solana-vote-program = { path = "../programs/vote", version = "1.3.4" }
+solana-account-decoder = { path = "../account-decoder", version = "1.3.6" }
+solana-net-utils = { path = "../net-utils", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-transaction-status = { path = "../transaction-status", version = "1.3.6" }
+solana-vote-program = { path = "../programs/vote", version = "1.3.6" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -32,7 +32,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "14.2.0"
 jsonrpc-http-server = "14.2.0"
-solana-logger = { path = "../logger", version = "1.3.4" }
+solana-logger = { path = "../logger", version = "1.3.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/rpc_config.rs

@@ -12,12 +12,14 @@ pub struct RpcSignatureStatusConfig {
 #[serde(rename_all = "camelCase")]
 pub struct RpcSendTransactionConfig {
     pub skip_preflight: bool,
+    pub preflight_commitment: Option<CommitmentConfig>,
 }
 
 #[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcSimulateTransactionConfig {
     pub sig_verify: bool,
+    pub commitment: Option<CommitmentConfig>,
 }
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.3.4"
+version = "1.3.6"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -43,38 +43,39 @@ regex = "1.3.9"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.3.4" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.3.4" }
-solana-budget-program = { path = "../programs/budget", version = "1.3.4" }
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-client = { path = "../client", version = "1.3.4" }
-solana-faucet = { path = "../faucet", version = "1.3.4" }
-solana-genesis-programs = { path = "../genesis-programs", version = "1.3.4" }
-solana-ledger = { path = "../ledger", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.3.4" }
-solana-metrics = { path = "../metrics", version = "1.3.4" }
-solana-measure = { path = "../measure", version = "1.3.4" }
-solana-net-utils = { path = "../net-utils", version = "1.3.4" }
-solana-perf = { path = "../perf", version = "1.3.4" }
-solana-runtime = { path = "../runtime", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-sdk-macro-frozen-abi = { path = "../sdk/macro-frozen-abi", version = "1.3.4" }
-solana-stake-program = { path = "../programs/stake", version = "1.3.4" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "1.3.4" }
-solana-streamer = { path = "../streamer", version = "1.3.4" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.3.4" }
-solana-transaction-status = { path = "../transaction-status", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
-solana-vote-program = { path = "../programs/vote", version = "1.3.4" }
-solana-vote-signer = { path = "../vote-signer", version = "1.3.4" }
-spl-token-v1-0 = { package = "spl-token", version = "1.0.6", features = ["skip-no-mangle"] }
+solana-account-decoder = { path = "../account-decoder", version = "1.3.6" }
+solana-banks-server = { path = "../banks-server", version = "1.3.6" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.3.6" }
+solana-budget-program = { path = "../programs/budget", version = "1.3.6" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-client = { path = "../client", version = "1.3.6" }
+solana-faucet = { path = "../faucet", version = "1.3.6" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.3.6" }
+solana-ledger = { path = "../ledger", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.3.6" }
+solana-metrics = { path = "../metrics", version = "1.3.6" }
+solana-measure = { path = "../measure", version = "1.3.6" }
+solana-net-utils = { path = "../net-utils", version = "1.3.6" }
+solana-perf = { path = "../perf", version = "1.3.6" }
+solana-runtime = { path = "../runtime", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-sdk-macro-frozen-abi = { path = "../sdk/macro-frozen-abi", version = "1.3.6" }
+solana-stake-program = { path = "../programs/stake", version = "1.3.6" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "1.3.6" }
+solana-streamer = { path = "../streamer", version = "1.3.6" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.3.6" }
+solana-transaction-status = { path = "../transaction-status", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
+solana-vote-program = { path = "../programs/vote", version = "1.3.6" }
+solana-vote-signer = { path = "../vote-signer", version = "1.3.6" }
+spl-token-v2-0 = { package = "spl-token", version = "2.0.3", features = ["skip-no-mangle"] }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio_01 = { version = "0.1", package = "tokio" }
 tokio_fs_01 = { version = "0.1", package = "tokio-fs" }
 tokio_io_01 = { version = "0.1", package = "tokio-io" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.3.4" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.3.6" }
 tokio = { version = "0.2.22", features = ["full"] }
 trees = "0.2.1"
 

core/src/cluster_info.rs

@@ -358,7 +358,7 @@ pub fn make_accounts_hashes_message(
 }
 
 // TODO These messages should go through the gpu pipeline for spam filtering
-#[frozen_abi(digest = "6qRS1ZwydpdSqzeyRdDvn5uwfDdFYkuUz4K4jSkd1oFW")]
+#[frozen_abi(digest = "CnN1gW2K2TRydGc84eYnQJwdTADPjQf6LJLZ4RP1QeoH")]
 #[derive(Serialize, Deserialize, Debug, AbiEnumVisitor, AbiExample)]
 #[allow(clippy::large_enum_variant)]
 enum Protocol {
@@ -558,7 +558,7 @@ impl ClusterInfo {
                     }
                     let ip_addr = node.gossip.ip();
                     Some(format!(
-                        "{:15} {:2}| {:5} | {:44} |{:^15}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {}\n",
+                        "{:15} {:2}| {:5} | {:44} |{:^15}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {}\n",
                         if ContactInfo::is_valid_address(&node.gossip) {
                             ip_addr.to_string()
                         } else {
@@ -581,6 +581,7 @@ impl ClusterInfo {
                         addr_to_string(&ip_addr, &node.serve_repair),
                         addr_to_string(&ip_addr, &node.rpc),
                         addr_to_string(&ip_addr, &node.rpc_pubsub),
+                        addr_to_string(&ip_addr, &node.rpc_banks),
                         node.shred_version,
                     ))
                 }
@@ -2453,10 +2454,12 @@ impl Node {
         let rpc_pubsub_port = find_available_port_in_range(bind_ip_addr, (1024, 65535)).unwrap();
         let rpc_pubsub_addr =
             SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), rpc_pubsub_port);
+        let rpc_banks_port = find_available_port_in_range(bind_ip_addr, (1024, 65535)).unwrap();
+        let rpc_banks_addr =
+            SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), rpc_banks_port);
 
         let broadcast = vec![UdpSocket::bind("0.0.0.0:0").unwrap()];
         let retransmit_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
-        let unused = UdpSocket::bind("0.0.0.0:0").unwrap();
         let serve_repair = UdpSocket::bind("127.0.0.1:0").unwrap();
         let info = ContactInfo {
             id: *pubkey,
@@ -2466,7 +2469,7 @@ impl Node {
             repair: repair.local_addr().unwrap(),
             tpu: tpu.local_addr().unwrap(),
             tpu_forwards: tpu_forwards.local_addr().unwrap(),
-            unused: unused.local_addr().unwrap(),
+            rpc_banks: rpc_banks_addr,
             rpc: rpc_addr,
             rpc_pubsub: rpc_pubsub_addr,
             serve_repair: serve_repair.local_addr().unwrap(),
@@ -2547,7 +2550,7 @@ impl Node {
             repair: SocketAddr::new(gossip_addr.ip(), repair_port),
             tpu: SocketAddr::new(gossip_addr.ip(), tpu_port),
             tpu_forwards: SocketAddr::new(gossip_addr.ip(), tpu_forwards_port),
-            unused: socketaddr_any!(),
+            rpc_banks: socketaddr_any!(),
             rpc: socketaddr_any!(),
             rpc_pubsub: socketaddr_any!(),
             serve_repair: SocketAddr::new(gossip_addr.ip(), serve_repair_port),

core/src/commitment_service.rs

@@ -246,10 +246,7 @@ mod tests {
         bank_forks::BankForks,
         genesis_utils::{create_genesis_config_with_vote_accounts, ValidatorVoteKeypairs},
     };
-    use solana_sdk::{
-        pubkey::Pubkey,
-        signature::{Keypair, Signer},
-    };
+    use solana_sdk::{pubkey::Pubkey, signature::Signer};
     use solana_stake_program::stake_state;
     use solana_vote_program::{
         vote_state::{self, VoteStateVersions},
@@ -488,14 +485,8 @@ mod tests {
 
         let block_commitment_cache = RwLock::new(BlockCommitmentCache::new_for_tests());
 
-        let node_keypair = Arc::new(Keypair::new());
-        let vote_keypair = Arc::new(Keypair::new());
-        let stake_keypair = Arc::new(Keypair::new());
-        let validator_keypairs = vec![ValidatorVoteKeypairs {
-            node_keypair: node_keypair.clone(),
-            vote_keypair: vote_keypair.clone(),
-            stake_keypair,
-        }];
+        let validator_vote_keypairs = ValidatorVoteKeypairs::new_rand();
+        let validator_keypairs = vec![&validator_vote_keypairs];
         let GenesisConfigInfo {
             genesis_config,
             mint_keypair: _,
@@ -518,9 +509,9 @@ mod tests {
                 vec![x],
                 previous_bank.hash(),
                 previous_bank.last_blockhash(),
-                &node_keypair,
-                &vote_keypair,
-                &vote_keypair,
+                &validator_vote_keypairs.node_keypair,
+                &validator_vote_keypairs.vote_keypair,
+                &validator_vote_keypairs.vote_keypair,
                 None,
             );
             bank.process_transaction(&vote).unwrap();
@@ -528,7 +519,10 @@ mod tests {
         }
 
         let working_bank = bank_forks.working_bank();
-        let root = get_vote_account_root_slot(vote_keypair.pubkey(), &working_bank);
+        let root = get_vote_account_root_slot(
+            validator_vote_keypairs.vote_keypair.pubkey(),
+            &working_bank,
+        );
         for x in 0..root {
             bank_forks.set_root(x, &None, None);
         }
@@ -540,16 +534,19 @@ mod tests {
             vec![33],
             bank33.hash(),
             bank33.last_blockhash(),
-            &node_keypair,
-            &vote_keypair,
-            &vote_keypair,
+            &validator_vote_keypairs.node_keypair,
+            &validator_vote_keypairs.vote_keypair,
+            &validator_vote_keypairs.vote_keypair,
             None,
         );
         bank34.process_transaction(&vote33).unwrap();
         bank_forks.insert(bank34);
 
         let working_bank = bank_forks.working_bank();
-        let root = get_vote_account_root_slot(vote_keypair.pubkey(), &working_bank);
+        let root = get_vote_account_root_slot(
+            validator_vote_keypairs.vote_keypair.pubkey(),
+            &working_bank,
+        );
         let ancestors = working_bank.status_cache_ancestors();
         let _ = AggregateCommitmentService::update_commitment_cache(
             &block_commitment_cache,
@@ -601,9 +598,9 @@ mod tests {
                 vec![x],
                 previous_bank.hash(),
                 previous_bank.last_blockhash(),
-                &node_keypair,
-                &vote_keypair,
-                &vote_keypair,
+                &validator_vote_keypairs.node_keypair,
+                &validator_vote_keypairs.vote_keypair,
+                &validator_vote_keypairs.vote_keypair,
                 None,
             );
             bank.process_transaction(&vote).unwrap();
@@ -611,7 +608,10 @@ mod tests {
         }
 
         let working_bank = bank_forks.working_bank();
-        let root = get_vote_account_root_slot(vote_keypair.pubkey(), &working_bank);
+        let root = get_vote_account_root_slot(
+            validator_vote_keypairs.vote_keypair.pubkey(),
+            &working_bank,
+        );
         let ancestors = working_bank.status_cache_ancestors();
         let _ = AggregateCommitmentService::update_commitment_cache(
             &block_commitment_cache,

core/src/consensus.rs

@@ -18,7 +18,7 @@ use solana_vote_program::{
     },
 };
 use std::{
-    collections::{BTreeMap, HashMap, HashSet},
+    collections::{HashMap, HashSet},
     ops::Bound::{Included, Unbounded},
     sync::{Arc, RwLock},
 };
@@ -128,9 +128,9 @@ impl Tower {
 
     pub(crate) fn collect_vote_lockouts<F>(
         node_pubkey: &Pubkey,
-        bank_slot: u64,
+        bank_slot: Slot,
         vote_accounts: F,
-        ancestors: &HashMap<Slot, HashSet<u64>>,
+        ancestors: &HashMap<Slot, HashSet<Slot>>,
         all_pubkeys: &mut PubkeyReferences,
     ) -> ComputedBankState
     where
@@ -141,13 +141,13 @@ impl Tower {
         let mut bank_weight = 0;
         // Tree of intervals of lockouts of the form [slot, slot + slot.lockout],
         // keyed by end of the range
-        let mut lockout_intervals = BTreeMap::new();
+        let mut lockout_intervals = LockoutIntervals::new();
         let mut pubkey_votes = vec![];
-        for (key, (lamports, account)) in vote_accounts {
-            if lamports == 0 {
+        for (key, (voted_stake, account)) in vote_accounts {
+            if voted_stake == 0 {
                 continue;
             }
-            trace!("{} {} with stake {}", node_pubkey, key, lamports);
+            trace!("{} {} with stake {}", node_pubkey, key, voted_stake);
             let vote_state = VoteState::from(&account);
             if vote_state.is_none() {
                 datapoint_warn!(
@@ -197,7 +197,7 @@ impl Tower {
             vote_state.process_slot_vote_unchecked(bank_slot);
 
             for vote in &vote_state.votes {
-                bank_weight += vote.lockout() as u128 * lamports as u128;
+                bank_weight += vote.lockout() as u128 * voted_stake as u128;
                 Self::populate_ancestor_voted_stakes(&mut voted_stakes, &vote, ancestors);
             }
 
@@ -208,7 +208,7 @@ impl Tower {
                         slot: root,
                     };
                     trace!("ROOT: {}", vote.slot);
-                    bank_weight += vote.lockout() as u128 * lamports as u128;
+                    bank_weight += vote.lockout() as u128 * voted_stake as u128;
                     Self::populate_ancestor_voted_stakes(&mut voted_stakes, &vote, ancestors);
                 }
             }
@@ -217,7 +217,7 @@ impl Tower {
                     confirmation_count: MAX_LOCKOUT_HISTORY as u32,
                     slot: root,
                 };
-                bank_weight += vote.lockout() as u128 * lamports as u128;
+                bank_weight += vote.lockout() as u128 * voted_stake as u128;
                 Self::populate_ancestor_voted_stakes(&mut voted_stakes, &vote, ancestors);
             }
 
@@ -239,11 +239,11 @@ impl Tower {
                 Self::update_ancestor_voted_stakes(
                     &mut voted_stakes,
                     vote.slot,
-                    lamports,
+                    voted_stake,
                     ancestors,
                 );
             }
-            total_stake += lamports;
+            total_stake += voted_stake;
         }
 
         ComputedBankState {
@@ -473,8 +473,12 @@ impl Tower {
                         .lockout_intervals;
                     // Find any locked out intervals in this bank with endpoint >= last_vote,
                     // implies they are locked out at last_vote
-                    for (_lockout_interval_end, value) in lockout_intervals.range((Included(last_voted_slot), Unbounded)) {
-                        for (lockout_interval_start, vote_account_pubkey) in value {
+                    for (_lockout_interval_end, intervals_keyed_by_end) in lockout_intervals.range((Included(last_voted_slot), Unbounded)) {
+                        for (lockout_interval_start, vote_account_pubkey) in intervals_keyed_by_end {
+                            if locked_out_vote_accounts.contains(vote_account_pubkey) {
+                                continue;
+                            }
+
                             // Only count lockouts on slots that are:
                             // 1) Not ancestors of `last_vote`
                             // 2) Not from before the current root as we can't determine if
@@ -485,7 +489,6 @@ impl Tower {
                                 // is an ancestor of the current root, because `candidate_slot` is a
                                 // descendant of the current root
                                 && *lockout_interval_start > root
-                                && !locked_out_vote_accounts.contains(vote_account_pubkey)
                             {
                                 let stake = epoch_vote_accounts
                                     .get(vote_account_pubkey)
@@ -587,21 +590,21 @@ impl Tower {
     /// Note, stake is the same for all the ancestor.
     fn update_ancestor_voted_stakes(
         voted_stakes: &mut VotedStakes,
-        slot: Slot,
-        lamports: u64,
+        voted_slot: Slot,
+        voted_stake: u64,
         ancestors: &HashMap<Slot, HashSet<Slot>>,
     ) {
         // If there's no ancestors, that means this slot must be from
         // before the current root, so ignore this slot
-        let vote_slot_ancestors = ancestors.get(&slot);
+        let vote_slot_ancestors = ancestors.get(&voted_slot);
         if vote_slot_ancestors.is_none() {
             return;
         }
-        let mut slot_with_ancestors = vec![slot];
+        let mut slot_with_ancestors = vec![voted_slot];
         slot_with_ancestors.extend(vote_slot_ancestors.unwrap());
         for slot in slot_with_ancestors {
             let current = voted_stakes.entry(slot).or_default();
-            *current += lamports;
+            *current += voted_stake;
         }
     }
 

core/src/contact_info.rs

@@ -25,8 +25,8 @@ pub struct ContactInfo {
     pub tpu: SocketAddr,
     /// address to forward unprocessed transactions to
     pub tpu_forwards: SocketAddr,
-    /// unused address
-    pub unused: SocketAddr,
+    /// address to which to send bank state requests
+    pub rpc_banks: SocketAddr,
     /// address to which to send JSON-RPC requests
     pub rpc: SocketAddr,
     /// websocket for JSON-RPC push notifications
@@ -95,7 +95,7 @@ impl Default for ContactInfo {
             repair: socketaddr_any!(),
             tpu: socketaddr_any!(),
             tpu_forwards: socketaddr_any!(),
-            unused: socketaddr_any!(),
+            rpc_banks: socketaddr_any!(),
             rpc: socketaddr_any!(),
             rpc_pubsub: socketaddr_any!(),
             serve_repair: socketaddr_any!(),
@@ -115,7 +115,7 @@ impl ContactInfo {
             repair: socketaddr!("127.0.0.1:1237"),
             tpu: socketaddr!("127.0.0.1:1238"),
             tpu_forwards: socketaddr!("127.0.0.1:1239"),
-            unused: socketaddr!("127.0.0.1:1240"),
+            rpc_banks: socketaddr!("127.0.0.1:1240"),
             rpc: socketaddr!("127.0.0.1:1241"),
             rpc_pubsub: socketaddr!("127.0.0.1:1242"),
             serve_repair: socketaddr!("127.0.0.1:1243"),
@@ -137,7 +137,7 @@ impl ContactInfo {
             repair: addr,
             tpu: addr,
             tpu_forwards: addr,
-            unused: addr,
+            rpc_banks: addr,
             rpc: addr,
             rpc_pubsub: addr,
             serve_repair: addr,
@@ -162,6 +162,7 @@ impl ContactInfo {
         let repair = next_port(&bind_addr, 5);
         let rpc = SocketAddr::new(bind_addr.ip(), rpc_port::DEFAULT_RPC_PORT);
         let rpc_pubsub = SocketAddr::new(bind_addr.ip(), rpc_port::DEFAULT_RPC_PUBSUB_PORT);
+        let rpc_banks = SocketAddr::new(bind_addr.ip(), rpc_port::DEFAULT_RPC_BANKS_PORT);
         let serve_repair = next_port(&bind_addr, 6);
         Self {
             id: *pubkey,
@@ -171,7 +172,7 @@ impl ContactInfo {
             repair,
             tpu,
             tpu_forwards,
-            unused: "0.0.0.0:0".parse().unwrap(),
+            rpc_banks,
             rpc,
             rpc_pubsub,
             serve_repair,
@@ -248,7 +249,7 @@ mod tests {
         assert!(ci.rpc.ip().is_unspecified());
         assert!(ci.rpc_pubsub.ip().is_unspecified());
         assert!(ci.tpu.ip().is_unspecified());
-        assert!(ci.unused.ip().is_unspecified());
+        assert!(ci.rpc_banks.ip().is_unspecified());
         assert!(ci.serve_repair.ip().is_unspecified());
     }
     #[test]
@@ -260,7 +261,7 @@ mod tests {
         assert!(ci.rpc.ip().is_multicast());
         assert!(ci.rpc_pubsub.ip().is_multicast());
         assert!(ci.tpu.ip().is_multicast());
-        assert!(ci.unused.ip().is_multicast());
+        assert!(ci.rpc_banks.ip().is_multicast());
         assert!(ci.serve_repair.ip().is_multicast());
     }
     #[test]
@@ -273,7 +274,7 @@ mod tests {
         assert!(ci.rpc.ip().is_unspecified());
         assert!(ci.rpc_pubsub.ip().is_unspecified());
         assert!(ci.tpu.ip().is_unspecified());
-        assert!(ci.unused.ip().is_unspecified());
+        assert!(ci.rpc_banks.ip().is_unspecified());
         assert!(ci.serve_repair.ip().is_unspecified());
     }
     #[test]
@@ -286,7 +287,7 @@ mod tests {
         assert_eq!(ci.tpu_forwards.port(), 13);
         assert_eq!(ci.rpc.port(), rpc_port::DEFAULT_RPC_PORT);
         assert_eq!(ci.rpc_pubsub.port(), rpc_port::DEFAULT_RPC_PUBSUB_PORT);
-        assert!(ci.unused.ip().is_unspecified());
+        assert_eq!(ci.rpc_banks.port(), rpc_port::DEFAULT_RPC_BANKS_PORT);
         assert_eq!(ci.serve_repair.port(), 16);
     }
 
@@ -310,6 +311,10 @@ mod tests {
             d1.rpc_pubsub,
             socketaddr!(format!("127.0.0.1:{}", rpc_port::DEFAULT_RPC_PUBSUB_PORT))
         );
+        assert_eq!(
+            d1.rpc_banks,
+            socketaddr!(format!("127.0.0.1:{}", rpc_port::DEFAULT_RPC_BANKS_PORT))
+        );
         assert_eq!(d1.tvu_forwards, socketaddr!("127.0.0.1:1238"));
         assert_eq!(d1.repair, socketaddr!("127.0.0.1:1239"));
         assert_eq!(d1.serve_repair, socketaddr!("127.0.0.1:1240"));

core/src/crds_gossip_pull.rs

@@ -390,7 +390,8 @@ impl CrdsGossipPull {
         let past = now.saturating_sub(msg_timeout);
         let recent: Vec<_> = filters
             .iter()
-            .filter(|(caller, _)| caller.wallclock() < future && caller.wallclock() >= past)
+            .enumerate()
+            .filter(|(_, (caller, _))| caller.wallclock() < future && caller.wallclock() >= past)
             .collect();
         inc_new_counter_info!(
             "gossip_filter_crds_values-dropped_requests",
@@ -402,11 +403,13 @@ impl CrdsGossipPull {
         let mut total_skipped = 0;
         let mask_ones: Vec<_> = recent
             .iter()
-            .map(|(_caller, filter)| (!0u64).checked_shr(filter.mask_bits).unwrap_or(!0u64))
+            .map(|(_i, (_caller, filter))| (!0u64).checked_shr(filter.mask_bits).unwrap_or(!0u64))
             .collect();
         for (label, mask) in crds.masks.iter() {
-            recent.iter().zip(mask_ones.iter()).enumerate().for_each(
-                |(i, ((caller, filter), mask_ones))| {
+            recent
+                .iter()
+                .zip(mask_ones.iter())
+                .for_each(|((i, (caller, filter)), mask_ones)| {
                     if filter.test_mask_u64(*mask, *mask_ones) {
                         let item = crds.table.get(label).unwrap();
 
@@ -419,11 +422,10 @@ impl CrdsGossipPull {
                         }
 
                         if !filter.filter_contains(&item.value_hash) {
-                            ret[i].push(item.value.clone());
+                            ret[*i].push(item.value.clone());
                         }
                     }
-                },
-            );
+                });
         }
         inc_new_counter_info!("gossip_filter_crds_values-dropped_values", total_skipped);
         ret
@@ -727,15 +729,19 @@ mod test {
             dest.generate_pull_responses(&dest_crds, &filters, CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS);
         assert_eq!(rsp[0].len(), 0);
 
+        assert_eq!(filters.len(), 1);
+        filters.push(filters[0].clone());
         //should return new value since caller is new
-        filters[0].0 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
+        filters[1].0 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
             &Pubkey::new_rand(),
             CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS + 1,
         )));
 
         let rsp =
             dest.generate_pull_responses(&dest_crds, &filters, CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS);
-        assert_eq!(rsp[0].len(), 1);
+        assert_eq!(rsp.len(), 2);
+        assert_eq!(rsp[0].len(), 0);
+        assert_eq!(rsp[1].len(), 1); // Orders are also preserved.
     }
 
     #[test]

core/src/gossip_service.rs

@@ -69,6 +69,7 @@ pub fn discover_cluster(
     num_nodes: usize,
 ) -> std::io::Result<Vec<ContactInfo>> {
     discover(
+        None,
         Some(entrypoint),
         Some(num_nodes),
         Some(30),
@@ -81,6 +82,7 @@ pub fn discover_cluster(
 }
 
 pub fn discover(
+    keypair: Option<Arc<Keypair>>,
     entrypoint: Option<&SocketAddr>,
     num_nodes: Option<usize>, // num_nodes only counts validators, excludes spy nodes
     timeout: Option<u64>,
@@ -89,9 +91,11 @@ pub fn discover(
     my_gossip_addr: Option<&SocketAddr>,
     my_shred_version: u16,
 ) -> std::io::Result<(Vec<ContactInfo>, Vec<ContactInfo>)> {
+    let keypair = keypair.unwrap_or_else(|| Arc::new(Keypair::new()));
+
     let exit = Arc::new(AtomicBool::new(false));
     let (gossip_service, ip_echo, spy_ref) =
-        make_gossip_node(entrypoint, &exit, my_gossip_addr, my_shred_version);
+        make_gossip_node(keypair, entrypoint, &exit, my_gossip_addr, my_shred_version);
 
     let id = spy_ref.id();
     info!("Entrypoint: {:?}", entrypoint);
@@ -245,12 +249,12 @@ fn spy(
 /// Makes a spy or gossip node based on whether or not a gossip_addr was passed in
 /// Pass in a gossip addr to fully participate in gossip instead of relying on just pulls
 fn make_gossip_node(
+    keypair: Arc<Keypair>,
     entrypoint: Option<&SocketAddr>,
     exit: &Arc<AtomicBool>,
     gossip_addr: Option<&SocketAddr>,
     shred_version: u16,
 ) -> (GossipService, Option<TcpListener>, Arc<ClusterInfo>) {
-    let keypair = Arc::new(Keypair::new());
     let (node, gossip_socket, ip_echo) = if let Some(gossip_addr) = gossip_addr {
         ClusterInfo::gossip_node(&keypair.pubkey(), gossip_addr, shred_version)
     } else {

core/src/lib.rs

@@ -72,9 +72,6 @@ pub mod vote_stake_tracker;
 pub mod weighted_shuffle;
 pub mod window_service;
 
-#[macro_use]
-extern crate solana_bpf_loader_program;
-
 #[macro_use]
 extern crate solana_budget_program;
 

core/src/repair_service.rs

@@ -18,7 +18,7 @@ use solana_measure::measure::Measure;
 use solana_runtime::{bank::Bank, bank_forks::BankForks, commitment::VOTE_THRESHOLD_SIZE};
 use solana_sdk::{clock::Slot, epoch_schedule::EpochSchedule, pubkey::Pubkey, timing::timestamp};
 use std::{
-    collections::HashMap,
+    collections::{HashMap, HashSet},
     iter::Iterator,
     net::SocketAddr,
     net::UdpSocket,
@@ -108,6 +108,7 @@ pub struct RepairInfo {
     pub bank_forks: Arc<RwLock<BankForks>>,
     pub epoch_schedule: EpochSchedule,
     pub duplicate_slots_reset_sender: DuplicateSlotsResetSender,
+    pub repair_validators: Option<HashSet<Pubkey>>,
 }
 
 pub struct RepairSlotRange {
@@ -234,6 +235,7 @@ impl RepairService {
                     blockstore,
                     &serve_repair,
                     &repair_info.duplicate_slots_reset_sender,
+                    &repair_info.repair_validators,
                 );
                 Self::generate_and_send_duplicate_repairs(
                     &mut duplicate_slot_repair_statuses,
@@ -242,6 +244,7 @@ impl RepairService {
                     &serve_repair,
                     &mut repair_stats,
                     &repair_socket,
+                    &repair_info.repair_validators,
                 );*/
 
                 repair_weight.get_best_weighted_repairs(
@@ -263,6 +266,7 @@ impl RepairService {
                     repair_request,
                     &mut cache,
                     &mut repair_stats,
+                    &repair_info.repair_validators,
                 ) {
                     repair_socket.send_to(&req, to).unwrap_or_else(|e| {
                         info!("{} repair req send_to({}) error {:?}", id, to, e);
@@ -444,9 +448,16 @@ impl RepairService {
         serve_repair: &ServeRepair,
         repair_stats: &mut RepairStats,
         repair_socket: &UdpSocket,
+        repair_validators: &Option<HashSet<Pubkey>>,
     ) {
         duplicate_slot_repair_statuses.retain(|slot, status| {
-            Self::update_duplicate_slot_repair_addr(*slot, status, cluster_slots, serve_repair);
+            Self::update_duplicate_slot_repair_addr(
+                *slot,
+                status,
+                cluster_slots,
+                serve_repair,
+                repair_validators,
+            );
             if let Some((repair_pubkey, repair_addr)) = status.repair_pubkey_and_addr {
                 let repairs = Self::generate_duplicate_repairs_for_slot(&blockstore, *slot);
 
@@ -499,13 +510,17 @@ impl RepairService {
         status: &mut DuplicateSlotRepairStatus,
         cluster_slots: &ClusterSlots,
         serve_repair: &ServeRepair,
+        repair_validators: &Option<HashSet<Pubkey>>,
     ) {
         let now = timestamp();
         if status.repair_pubkey_and_addr.is_none()
             || now.saturating_sub(status.start) >= MAX_DUPLICATE_WAIT_MS as u64
         {
-            let repair_pubkey_and_addr =
-                serve_repair.repair_request_duplicate_compute_best_peer(slot, cluster_slots);
+            let repair_pubkey_and_addr = serve_repair.repair_request_duplicate_compute_best_peer(
+                slot,
+                cluster_slots,
+                repair_validators,
+            );
             status.repair_pubkey_and_addr = repair_pubkey_and_addr.ok();
             status.start = timestamp();
         }
@@ -520,6 +535,7 @@ impl RepairService {
         blockstore: &Blockstore,
         serve_repair: &ServeRepair,
         duplicate_slots_reset_sender: &DuplicateSlotsResetSender,
+        repair_validators: &Option<HashSet<Pubkey>>,
     ) {
         for slot in new_duplicate_slots {
             warn!(
@@ -545,7 +561,7 @@ impl RepairService {
             // Mark this slot as special repair, try to download from single
             // validator to avoid corruption
             let repair_pubkey_and_addr = serve_repair
-                .repair_request_duplicate_compute_best_peer(*slot, cluster_slots)
+                .repair_request_duplicate_compute_best_peer(*slot, cluster_slots, repair_validators)
                 .ok();
             let new_duplicate_slot_repair_status = DuplicateSlotRepairStatus {
                 start: timestamp(),
@@ -953,6 +969,7 @@ mod test {
             &serve_repair,
             &mut RepairStats::default(),
             &UdpSocket::bind("0.0.0.0:0").unwrap(),
+            &None,
         );
         assert!(duplicate_slot_repair_statuses
             .get(&dead_slot)
@@ -976,6 +993,7 @@ mod test {
             &serve_repair,
             &mut RepairStats::default(),
             &UdpSocket::bind("0.0.0.0:0").unwrap(),
+            &None,
         );
         assert_eq!(duplicate_slot_repair_statuses.len(), 1);
         assert!(duplicate_slot_repair_statuses.get(&dead_slot).is_some());
@@ -992,6 +1010,7 @@ mod test {
             &serve_repair,
             &mut RepairStats::default(),
             &UdpSocket::bind("0.0.0.0:0").unwrap(),
+            &None,
         );
         assert!(duplicate_slot_repair_statuses.is_empty());
     }
@@ -1026,6 +1045,7 @@ mod test {
             &mut duplicate_status,
             &cluster_slots,
             &serve_repair,
+            &None,
         );
         assert_eq!(duplicate_status.repair_pubkey_and_addr, dummy_addr);
 
@@ -1039,6 +1059,7 @@ mod test {
             &mut duplicate_status,
             &cluster_slots,
             &serve_repair,
+            &None,
         );
         assert!(duplicate_status.repair_pubkey_and_addr.is_some());
 
@@ -1052,6 +1073,7 @@ mod test {
             &mut duplicate_status,
             &cluster_slots,
             &serve_repair,
+            &None,
         );
         assert_ne!(duplicate_status.repair_pubkey_and_addr, dummy_addr);
     }
@@ -1108,6 +1130,7 @@ mod test {
             &blockstore,
             &serve_repair,
             &reset_sender,
+            &None,
         );
 
         // Blockstore should have been cleared

core/src/repair_weight.rs

@@ -511,6 +511,7 @@ mod test {
     use super::*;
     use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
     use solana_runtime::{bank::Bank, bank_utils};
+    use solana_sdk::hash::Hash;
     use trees::tr;
 
     #[test]
@@ -658,7 +659,7 @@ mod test {
         assert_eq!(repair_weight.trees.get(&8).unwrap().ancestors(10), vec![8]);
 
         // Connect orphan back to main fork
-        blockstore.add_tree(tr(6) / (tr(8)), true, true);
+        blockstore.add_tree(tr(6) / (tr(8)), true, true, 2, Hash::default());
         assert_eq!(
             AncestorIterator::new(8, &blockstore).collect::<Vec<_>>(),
             vec![6, 5, 3, 1, 0]
@@ -742,8 +743,8 @@ mod test {
         assert!(repair_weight.trees.contains_key(&20));
 
         // Resolve orphans in blockstore
-        blockstore.add_tree(tr(6) / (tr(8)), true, true);
-        blockstore.add_tree(tr(11) / (tr(20)), true, true);
+        blockstore.add_tree(tr(6) / (tr(8)), true, true, 2, Hash::default());
+        blockstore.add_tree(tr(11) / (tr(20)), true, true, 2, Hash::default());
         // Call `update_orphan_ancestors` to resolve orphan
         repair_weight.update_orphan_ancestors(
             &blockstore,
@@ -853,8 +854,8 @@ mod test {
         // Resolve the orphans, should now return no
         // orphans
         repairs = vec![];
-        blockstore.add_tree(tr(6) / (tr(8)), true, true);
-        blockstore.add_tree(tr(11) / (tr(20)), true, true);
+        blockstore.add_tree(tr(6) / (tr(8)), true, true, 2, Hash::default());
+        blockstore.add_tree(tr(11) / (tr(20)), true, true, 2, Hash::default());
         repair_weight.get_best_orphans(
             &blockstore,
             &mut repairs,
@@ -889,7 +890,7 @@ mod test {
         // orphan in the `trees` map, we should search for
         // exactly one more of the remaining two
         let mut repairs = vec![];
-        blockstore.add_tree(tr(100) / (tr(101)), true, true);
+        blockstore.add_tree(tr(100) / (tr(101)), true, true, 2, Hash::default());
         repair_weight.get_best_orphans(
             &blockstore,
             &mut repairs,
@@ -991,7 +992,7 @@ mod test {
 
         // Chain orphan 8 back to the main fork, but don't
         // touch orphan 20
-        blockstore.add_tree(tr(4) / (tr(8)), true, true);
+        blockstore.add_tree(tr(4) / (tr(8)), true, true, 2, Hash::default());
 
         // Call `update_orphan_ancestors` to resolve orphan
         repair_weight.update_orphan_ancestors(
@@ -1061,10 +1062,10 @@ mod test {
             }
 
             // Chain orphan 8 back to slot `old_parent`
-            blockstore.add_tree(tr(*old_parent) / (tr(8)), true, true);
+            blockstore.add_tree(tr(*old_parent) / (tr(8)), true, true, 2, Hash::default());
 
             // Chain orphan 20 back to orphan 8
-            blockstore.add_tree(tr(8) / (tr(20)), true, true);
+            blockstore.add_tree(tr(8) / (tr(20)), true, true, 2, Hash::default());
 
             // Call `update_orphan_ancestors` to resolve orphan
             repair_weight.update_orphan_ancestors(
@@ -1089,7 +1090,13 @@ mod test {
 
             // Add a vote that chains back to `old_parent`, should be purged
             let new_vote_slot = 100;
-            blockstore.add_tree(tr(*old_parent) / tr(new_vote_slot), true, true);
+            blockstore.add_tree(
+                tr(*old_parent) / tr(new_vote_slot),
+                true,
+                true,
+                2,
+                Hash::default(),
+            );
             repair_weight.add_votes(
                 &blockstore,
                 vec![(new_vote_slot, vec![Pubkey::default()])].into_iter(),
@@ -1137,7 +1144,7 @@ mod test {
         );
 
         // Ancestors of slot 31 are [30], with no existing subtree
-        blockstore.add_tree(tr(30) / (tr(31)), true, true);
+        blockstore.add_tree(tr(30) / (tr(31)), true, true, 2, Hash::default());
         assert_eq!(
             repair_weight.find_ancestor_subtree_of_slot(&blockstore, 31),
             (vec![30].into_iter().collect::<VecDeque<_>>(), None)
@@ -1155,7 +1162,7 @@ mod test {
 
         // Chain orphan 8 back to slot 4 on a different fork, ancestor search
         // should not return ancestors earlier than the root
-        blockstore.add_tree(tr(4) / (tr(8)), true, true);
+        blockstore.add_tree(tr(4) / (tr(8)), true, true, 2, Hash::default());
         assert_eq!(
             repair_weight.find_ancestor_subtree_of_slot(&blockstore, 8),
             (vec![4].into_iter().collect::<VecDeque<_>>(), None)
@@ -1242,8 +1249,8 @@ mod test {
         */
 
         let blockstore = setup_forks();
-        blockstore.add_tree(tr(8) / (tr(10) / (tr(11))), true, true);
-        blockstore.add_tree(tr(20) / (tr(22) / (tr(23))), true, true);
+        blockstore.add_tree(tr(8) / (tr(10) / (tr(11))), true, true, 2, Hash::default());
+        blockstore.add_tree(tr(20) / (tr(22) / (tr(23))), true, true, 2, Hash::default());
         assert!(blockstore.orphan(8).unwrap().is_some());
         blockstore
     }
@@ -1265,7 +1272,7 @@ mod test {
         let forks = tr(0) / (tr(1) / (tr(2) / (tr(4))) / (tr(3) / (tr(5) / (tr(6)))));
         let ledger_path = get_tmp_ledger_path!();
         let blockstore = Blockstore::open(&ledger_path).unwrap();
-        blockstore.add_tree(forks, false, true);
+        blockstore.add_tree(forks, false, true, 2, Hash::default());
         blockstore
     }
 }

core/src/repair_weighted_traversal.rs

@@ -150,6 +150,7 @@ pub mod test {
     use super::*;
     use solana_ledger::{get_tmp_ledger_path, shred::Shred};
     use solana_runtime::bank_utils;
+    use solana_sdk::hash::Hash;
     use trees::tr;
 
     #[test]
@@ -246,7 +247,13 @@ pub mod test {
         repairs = vec![];
         let best_overall_slot = heaviest_subtree_fork_choice.best_overall_slot();
         assert_eq!(heaviest_subtree_fork_choice.best_overall_slot(), 4);
-        blockstore.add_tree(tr(best_overall_slot) / (tr(6) / tr(7)), true, false);
+        blockstore.add_tree(
+            tr(best_overall_slot) / (tr(6) / tr(7)),
+            true,
+            false,
+            2,
+            Hash::default(),
+        );
         get_best_repair_shreds(
             &heaviest_subtree_fork_choice,
             &blockstore,
@@ -300,7 +307,7 @@ pub mod test {
         // Adding incomplete children with higher weighted parents, even if
         // the parents are complete should still be repaired
         repairs = vec![];
-        blockstore.add_tree(tr(2) / (tr(8)), true, false);
+        blockstore.add_tree(tr(2) / (tr(8)), true, false, 2, Hash::default());
         get_best_repair_shreds(
             &heaviest_subtree_fork_choice,
             &blockstore,
@@ -322,7 +329,7 @@ pub mod test {
         let (blockstore, heaviest_subtree_fork_choice) = setup_forks();
         // Add a branch to slot 2, make sure it doesn't repair child
         // 4 again when the Unvisited(2) event happens
-        blockstore.add_tree(tr(2) / (tr(6) / tr(7)), true, false);
+        blockstore.add_tree(tr(2) / (tr(6) / tr(7)), true, false, 2, Hash::default());
         let mut repairs = vec![];
         get_best_repair_shreds(
             &heaviest_subtree_fork_choice,
@@ -368,7 +375,7 @@ pub mod test {
         // Adding slot 2 to ignore should not remove its unexplored children from
         // the repair set
         repairs = vec![];
-        blockstore.add_tree(tr(2) / (tr(6) / tr(7)), true, false);
+        blockstore.add_tree(tr(2) / (tr(6) / tr(7)), true, false, 2, Hash::default());
         ignore_set.insert(2);
         get_best_repair_shreds(
             &heaviest_subtree_fork_choice,
@@ -420,7 +427,7 @@ pub mod test {
         let forks = tr(0) / (tr(1) / (tr(2) / (tr(4))) / (tr(3) / (tr(5))));
         let ledger_path = get_tmp_ledger_path!();
         let blockstore = Blockstore::open(&ledger_path).unwrap();
-        blockstore.add_tree(forks.clone(), false, false);
+        blockstore.add_tree(forks.clone(), false, false, 2, Hash::default());
 
         (blockstore, HeaviestSubtreeForkChoice::new_from_tree(forks))
     }

core/src/retransmit_stage.rs

@@ -28,6 +28,7 @@ use solana_sdk::timing::timestamp;
 use solana_streamer::streamer::PacketReceiver;
 use std::{
     cmp,
+    collections::hash_set::HashSet,
     collections::{BTreeMap, HashMap},
     net::UdpSocket,
     sync::atomic::{AtomicBool, AtomicU64, Ordering},
@@ -417,6 +418,7 @@ impl RetransmitStage {
         cluster_slots: Arc<ClusterSlots>,
         duplicate_slots_reset_sender: DuplicateSlotsResetSender,
         verified_vote_receiver: VerifiedVoteReceiver,
+        repair_validators: Option<HashSet<Pubkey>>,
     ) -> Self {
         let (retransmit_sender, retransmit_receiver) = channel();
 
@@ -442,6 +444,7 @@ impl RetransmitStage {
             bank_forks,
             epoch_schedule,
             duplicate_slots_reset_sender,
+            repair_validators,
         };
         let window_service = WindowService::new(
             blockstore,

core/src/rpc.rs

@@ -11,7 +11,7 @@ use jsonrpc_derive::rpc;
 use solana_account_decoder::{
     parse_account_data::AccountAdditionalData,
     parse_token::{
-        get_token_account_mint, spl_token_id_v1_0, spl_token_v1_0_native_mint,
+        get_token_account_mint, spl_token_id_v2_0, spl_token_v2_0_native_mint,
         token_amount_to_ui_amount, UiTokenAmount,
     },
     UiAccount, UiAccountEncoding,
@@ -58,11 +58,13 @@ use solana_transaction_status::{
     ConfirmedBlock, ConfirmedTransaction, TransactionStatus, UiTransactionEncoding,
 };
 use solana_vote_program::vote_state::{VoteState, MAX_LOCKOUT_HISTORY};
-use spl_token_v1_0::state::{Account as TokenAccount, Mint};
+use spl_token_v2_0::{
+    pack::Pack,
+    state::{Account as TokenAccount, Mint},
+};
 use std::{
     cmp::{max, min},
     collections::{HashMap, HashSet},
-    mem::size_of,
     net::SocketAddr,
     str::FromStr,
     sync::{
@@ -247,7 +249,7 @@ impl JsonRpcRequestProcessor {
         check_slice_and_encoding(&encoding, config.data_slice.is_some())?;
         let mut response = None;
         if let Some(account) = bank.get_account(pubkey) {
-            if account.owner == spl_token_id_v1_0() && encoding == UiAccountEncoding::JsonParsed {
+            if account.owner == spl_token_id_v2_0() && encoding == UiAccountEncoding::JsonParsed {
                 response = Some(get_parsed_token_account(bank.clone(), pubkey, account));
             } else if (encoding == UiAccountEncoding::Binary
                 || encoding == UiAccountEncoding::Base58)
@@ -295,7 +297,7 @@ impl JsonRpcRequestProcessor {
         check_slice_and_encoding(&encoding, data_slice_config.is_some())?;
         let keyed_accounts = get_filtered_program_accounts(&bank, program_id, filters);
         let result =
-            if program_id == &spl_token_id_v1_0() && encoding == UiAccountEncoding::JsonParsed {
+            if program_id == &spl_token_id_v2_0() && encoding == UiAccountEncoding::JsonParsed {
                 get_parsed_token_accounts(bank, keyed_accounts).collect()
             } else {
                 keyed_accounts
@@ -630,7 +632,7 @@ impl JsonRpcRequestProcessor {
             self.check_slot_cleaned_up(&result, slot)?;
             Ok(result.ok())
         } else {
-            Ok(None)
+            Err(RpcCustomError::BlockNotAvailable { slot }.into())
         }
     }
 
@@ -702,7 +704,7 @@ impl JsonRpcRequestProcessor {
             self.check_slot_cleaned_up(&result, slot)?;
             Ok(result.ok().unwrap_or(None))
         } else {
-            Ok(None)
+            Err(RpcCustomError::BlockNotAvailable { slot }.into())
         }
     }
 
@@ -1025,16 +1027,14 @@ impl JsonRpcRequestProcessor {
             Error::invalid_params("Invalid param: could not find account".to_string())
         })?;
 
-        if account.owner != spl_token_id_v1_0() {
+        if account.owner != spl_token_id_v2_0() {
             return Err(Error::invalid_params(
-                "Invalid param: not a v1.0 Token account".to_string(),
+                "Invalid param: not a v2.0 Token account".to_string(),
             ));
         }
-        let mut data = account.data.to_vec();
-        let token_account =
-            spl_token_v1_0::state::unpack::<TokenAccount>(&mut data).map_err(|_| {
-                Error::invalid_params("Invalid param: not a v1.0 Token account".to_string())
-            })?;
+        let token_account = TokenAccount::unpack(&account.data).map_err(|_| {
+            Error::invalid_params("Invalid param: not a v2.0 Token account".to_string())
+        })?;
         let mint = &Pubkey::from_str(&token_account.mint.to_string())
             .expect("Token account mint should be convertible to Pubkey");
         let (_, decimals) = get_mint_owner_and_decimals(&bank, &mint)?;
@@ -1048,32 +1048,19 @@ impl JsonRpcRequestProcessor {
         commitment: Option<CommitmentConfig>,
     ) -> Result<RpcResponse<UiTokenAmount>> {
         let bank = self.bank(commitment);
-        let (mint_owner, decimals) = get_mint_owner_and_decimals(&bank, mint)?;
-        if mint_owner != spl_token_id_v1_0() {
+        let mint_account = bank.get_account(mint).ok_or_else(|| {
+            Error::invalid_params("Invalid param: could not find account".to_string())
+        })?;
+        if mint_account.owner != spl_token_id_v2_0() {
             return Err(Error::invalid_params(
-                "Invalid param: not a v1.0 Token mint".to_string(),
+                "Invalid param: not a v2.0 Token mint".to_string(),
             ));
         }
+        let mint = Mint::unpack(&mint_account.data).map_err(|_| {
+            Error::invalid_params("Invalid param: mint could not be unpacked".to_string())
+        })?;
 
-        let filters = vec![
-            // Filter on Mint address
-            RpcFilterType::Memcmp(Memcmp {
-                offset: 0,
-                bytes: MemcmpEncodedBytes::Binary(mint.to_string()),
-                encoding: None,
-            }),
-            // Filter on Token Account state
-            RpcFilterType::DataSize(size_of::<TokenAccount>() as u64),
-        ];
-        let supply = get_filtered_program_accounts(&bank, &mint_owner, filters)
-            .map(|(_pubkey, account)| {
-                let mut data = account.data.to_vec();
-                spl_token_v1_0::state::unpack(&mut data)
-                    .map(|account: &mut TokenAccount| account.amount)
-                    .unwrap_or(0)
-            })
-            .sum();
-        let supply = token_amount_to_ui_amount(supply, decimals);
+        let supply = token_amount_to_ui_amount(mint.supply, mint.decimals);
         Ok(new_response(&bank, supply))
     }
 
@@ -1084,9 +1071,9 @@ impl JsonRpcRequestProcessor {
     ) -> Result<RpcResponse<Vec<RpcTokenAccountBalance>>> {
         let bank = self.bank(commitment);
         let (mint_owner, decimals) = get_mint_owner_and_decimals(&bank, mint)?;
-        if mint_owner != spl_token_id_v1_0() {
+        if mint_owner != spl_token_id_v2_0() {
             return Err(Error::invalid_params(
-                "Invalid param: not a v1.0 Token mint".to_string(),
+                "Invalid param: not a v2.0 Token mint".to_string(),
             ));
         }
         let filters = vec![
@@ -1097,14 +1084,13 @@ impl JsonRpcRequestProcessor {
                 encoding: None,
             }),
             // Filter on Token Account state
-            RpcFilterType::DataSize(size_of::<TokenAccount>() as u64),
+            RpcFilterType::DataSize(TokenAccount::get_packed_len() as u64),
         ];
         let mut token_balances: Vec<RpcTokenAccountBalance> =
             get_filtered_program_accounts(&bank, &mint_owner, filters)
                 .map(|(address, account)| {
-                    let mut data = account.data.to_vec();
-                    let amount = spl_token_v1_0::state::unpack(&mut data)
-                        .map(|account: &mut TokenAccount| account.amount)
+                    let amount = TokenAccount::unpack(&account.data)
+                        .map(|account| account.amount)
                         .unwrap_or(0);
                     let amount = token_amount_to_ui_amount(amount, decimals);
                     RpcTokenAccountBalance {
@@ -1146,7 +1132,7 @@ impl JsonRpcRequestProcessor {
                 encoding: None,
             }),
             // Filter on Token Account state
-            RpcFilterType::DataSize(size_of::<TokenAccount>() as u64),
+            RpcFilterType::DataSize(TokenAccount::get_packed_len() as u64),
         ];
         if let Some(mint) = mint {
             // Optional filter on Mint address
@@ -1205,7 +1191,7 @@ impl JsonRpcRequestProcessor {
                 encoding: None,
             }),
             // Filter on Token Account state
-            RpcFilterType::DataSize(size_of::<TokenAccount>() as u64),
+            RpcFilterType::DataSize(TokenAccount::get_packed_len() as u64),
         ];
         if let Some(mint) = mint {
             // Optional filter on Mint address
@@ -1333,26 +1319,29 @@ where
     I: Iterator<Item = (Pubkey, Account)>,
 {
     let mut mint_decimals: HashMap<Pubkey, u8> = HashMap::new();
-    keyed_accounts.map(move |(pubkey, account)| {
-        let additional_data = get_token_account_mint(&account.data).map(|mint_pubkey| {
-            let spl_token_decimals = mint_decimals.get(&mint_pubkey).cloned().or_else(|| {
-                let (_, decimals) = get_mint_owner_and_decimals(&bank, &mint_pubkey).ok()?;
-                mint_decimals.insert(mint_pubkey, decimals);
-                Some(decimals)
+    keyed_accounts.filter_map(move |(pubkey, account)| {
+        let additional_data = get_token_account_mint(&account.data)
+            .and_then(|mint_pubkey| {
+                mint_decimals.get(&mint_pubkey).cloned().or_else(|| {
+                    let (_, decimals) = get_mint_owner_and_decimals(&bank, &mint_pubkey).ok()?;
+                    mint_decimals.insert(mint_pubkey, decimals);
+                    Some(decimals)
+                })
+            })
+            .map(|spl_token_decimals| AccountAdditionalData {
+                spl_token_decimals: Some(spl_token_decimals),
             });
-            AccountAdditionalData { spl_token_decimals }
-        });
 
-        RpcKeyedAccount {
+        additional_data.map(|additional_data| RpcKeyedAccount {
             pubkey: pubkey.to_string(),
             account: UiAccount::encode(
                 &pubkey,
                 account,
                 UiAccountEncoding::JsonParsed,
-                additional_data,
+                Some(additional_data),
                 None,
             ),
-        }
+        })
     })
 }
 
@@ -1365,15 +1354,15 @@ fn get_token_program_id_and_mint(
     match token_account_filter {
         TokenAccountsFilter::Mint(mint) => {
             let (mint_owner, _) = get_mint_owner_and_decimals(&bank, &mint)?;
-            if mint_owner != spl_token_id_v1_0() {
+            if mint_owner != spl_token_id_v2_0() {
                 return Err(Error::invalid_params(
-                    "Invalid param: not a v1.0 Token mint".to_string(),
+                    "Invalid param: not a v2.0 Token mint".to_string(),
                 ));
             }
             Ok((mint_owner, Some(mint)))
         }
         TokenAccountsFilter::ProgramId(program_id) => {
-            if program_id == spl_token_id_v1_0() {
+            if program_id == spl_token_id_v2_0() {
                 Ok((program_id, None))
             } else {
                 Err(Error::invalid_params(
@@ -1387,10 +1376,8 @@ fn get_token_program_id_and_mint(
 /// Analyze a mint Pubkey that may be the native_mint and get the mint-account owner (token
 /// program_id) and decimals
 fn get_mint_owner_and_decimals(bank: &Arc<Bank>, mint: &Pubkey) -> Result<(Pubkey, u8)> {
-    if mint == &spl_token_v1_0_native_mint() {
-        // Uncomment the following once spl_token is bumped to a version that includes native_mint::DECIMALS
-        // Ok((spl_token_id_v1_0(), spl_token_v1_0::native_mint::DECIMALS))
-        Ok((spl_token_id_v1_0(), 9))
+    if mint == &spl_token_v2_0_native_mint() {
+        Ok((spl_token_id_v2_0(), spl_token_v2_0::native_mint::DECIMALS))
     } else {
         let mint_account = bank.get_account(mint).ok_or_else(|| {
             Error::invalid_params("Invalid param: could not find mint".to_string())
@@ -1401,12 +1388,11 @@ fn get_mint_owner_and_decimals(bank: &Arc<Bank>, mint: &Pubkey) -> Result<(Pubke
 }
 
 fn get_mint_decimals(data: &[u8]) -> Result<u8> {
-    let mut data = data.to_vec();
-    spl_token_v1_0::state::unpack(&mut data)
+    Mint::unpack(data)
         .map_err(|_| {
             Error::invalid_params("Invalid param: Token mint could not be unpacked".to_string())
         })
-        .map(|mint: &mut Mint| mint.decimals)
+        .map(|mint| mint.decimals)
 }
 
 #[rpc]
@@ -1691,7 +1677,7 @@ pub trait RpcSol {
     ) -> Result<RpcStakeActivation>;
 
     // SPL Token-specific RPC endpoints
-    // See https://github.com/solana-labs/solana-program-library/releases/tag/token-v1.0.0 for
+    // See https://github.com/solana-labs/solana-program-library/releases/tag/token-v2.0.0 for
     // program details
 
     #[rpc(meta, name = "getTokenAccountBalance")]
@@ -2139,7 +2125,10 @@ impl RpcSol for RpcSolImpl {
                 .into());
             }
 
-            if let (Err(err), _log_output) = bank.simulate_transaction(transaction.clone()) {
+            let preflight_bank = &*meta.bank(config.preflight_commitment);
+            if let (Err(err), _log_output) =
+                preflight_bank.simulate_transaction(transaction.clone())
+            {
                 // Note: it's possible that the transaction simulation failed but the actual
                 // transaction would succeed, such as when a transaction depends on an earlier
                 // transaction that has yet to reach max confirmations. In these cases the user
@@ -2171,7 +2160,7 @@ impl RpcSol for RpcSolImpl {
             Ok(())
         };
 
-        let bank = &*meta.bank(None);
+        let bank = &*meta.bank(config.commitment);
         let logs = if result.is_ok() {
             let (transaction_result, log_messages) = bank.simulate_transaction(transaction);
             result = transaction_result;
@@ -2501,8 +2490,9 @@ pub mod tests {
         vote_instruction,
         vote_state::{Vote, VoteInit, MAX_LOCKOUT_HISTORY},
     };
-    use spl_token_v1_0::{
-        option::COption, solana_sdk::pubkey::Pubkey as SplTokenPubkey, state::Mint,
+    use spl_token_v2_0::{
+        option::COption, solana_sdk::pubkey::Pubkey as SplTokenPubkey,
+        state::AccountState as TokenAccountState, state::Mint,
     };
     use std::collections::HashMap;
 
@@ -4045,7 +4035,7 @@ pub mod tests {
         (
             Arc::new(RwLock::new(BankForks::new(bank))),
             mint_keypair,
-            voting_keypair,
+            Arc::new(voting_keypair),
         )
     }
 
@@ -4501,7 +4491,7 @@ pub mod tests {
             slot
         );
         let res = io.handle_request_sync(&req, meta);
-        let expected = r#"{"jsonrpc":"2.0","result":null,"id":1}"#;
+        let expected = r#"{"jsonrpc":"2.0","error":{"code":-32004,"message":"Block not available for slot 12345"},"id":1}"#;
         let expected: Response =
             serde_json::from_str(&expected).expect("expected response deserialization");
         let result: Response = serde_json::from_str(&res.expect("actual response"))
@@ -4757,43 +4747,50 @@ pub mod tests {
     fn test_token_rpcs() {
         let RpcHandler { io, meta, bank, .. } = start_rpc_handler_with_tx(&Pubkey::new_rand());
 
-        let mut account_data = [0; size_of::<TokenAccount>()];
-        let account: &mut TokenAccount =
-            spl_token_v1_0::state::unpack_unchecked(&mut account_data).unwrap();
+        let mut account_data = vec![0; TokenAccount::get_packed_len()];
         let mint = SplTokenPubkey::new(&[2; 32]);
         let owner = SplTokenPubkey::new(&[3; 32]);
         let delegate = SplTokenPubkey::new(&[4; 32]);
-        *account = TokenAccount {
-            mint,
-            owner,
-            delegate: COption::Some(delegate),
-            amount: 420,
-            is_initialized: true,
-            is_native: false,
-            delegated_amount: 30,
-        };
+        TokenAccount::unpack_unchecked_mut(&mut account_data, &mut |account: &mut TokenAccount| {
+            *account = TokenAccount {
+                mint,
+                owner,
+                delegate: COption::Some(delegate),
+                amount: 420,
+                state: TokenAccountState::Initialized,
+                is_native: COption::None,
+                delegated_amount: 30,
+                close_authority: COption::Some(owner),
+            };
+            Ok(())
+        })
+        .unwrap();
         let token_account = Account {
             lamports: 111,
             data: account_data.to_vec(),
-            owner: spl_token_id_v1_0(),
+            owner: spl_token_id_v2_0(),
             ..Account::default()
         };
         let token_account_pubkey = Pubkey::new_rand();
         bank.store_account(&token_account_pubkey, &token_account);
 
         // Add the mint
-        let mut mint_data = [0; size_of::<Mint>()];
-        let mint_state: &mut Mint =
-            spl_token_v1_0::state::unpack_unchecked(&mut mint_data).unwrap();
-        *mint_state = Mint {
-            owner: COption::Some(owner),
-            decimals: 2,
-            is_initialized: true,
-        };
+        let mut mint_data = vec![0; Mint::get_packed_len()];
+        Mint::unpack_unchecked_mut(&mut mint_data, &mut |mint: &mut Mint| {
+            *mint = Mint {
+                mint_authority: COption::Some(owner),
+                supply: 500,
+                decimals: 2,
+                is_initialized: true,
+                freeze_authority: COption::Some(owner),
+            };
+            Ok(())
+        })
+        .unwrap();
         let mint_account = Account {
             lamports: 111,
             data: mint_data.to_vec(),
-            owner: spl_token_id_v1_0(),
+            owner: spl_token_id_v2_0(),
             ..Account::default()
         };
         bank.store_account(&Pubkey::from_str(&mint.to_string()).unwrap(), &mint_account);
@@ -4822,10 +4819,7 @@ pub mod tests {
             .expect("actual response deserialization");
         assert!(result.get("error").is_some());
 
-        // Add another token account to ensure getTokenSupply sums all mint accounts
-        let other_token_account_pubkey = Pubkey::new_rand();
-        bank.store_account(&other_token_account_pubkey, &token_account);
-
+        // Test get token supply, pulls supply from mint
         let req = format!(
             r#"{{"jsonrpc":"2.0","id":1,"method":"getTokenSupply","params":["{}"]}}"#,
             mint,
@@ -4836,8 +4830,8 @@ pub mod tests {
         let supply: UiTokenAmount =
             serde_json::from_value(result["result"]["value"].clone()).unwrap();
         let error = f64::EPSILON;
-        assert!((supply.ui_amount - 2.0 * 4.2).abs() < error);
-        assert_eq!(supply.amount, (2 * 420).to_string());
+        assert!((supply.ui_amount - 5.0).abs() < error);
+        assert_eq!(supply.amount, 500.to_string());
         assert_eq!(supply.decimals, 2);
 
         // Test non-existent mint address
@@ -4850,24 +4844,31 @@ pub mod tests {
             .expect("actual response deserialization");
         assert!(result.get("error").is_some());
 
+        // Add another token account with the same owner, delegate, and mint
+        let other_token_account_pubkey = Pubkey::new_rand();
+        bank.store_account(&other_token_account_pubkey, &token_account);
+
         // Add another token account with the same owner and delegate but different mint
-        let mut account_data = [0; size_of::<TokenAccount>()];
-        let account: &mut TokenAccount =
-            spl_token_v1_0::state::unpack_unchecked(&mut account_data).unwrap();
+        let mut account_data = vec![0; TokenAccount::get_packed_len()];
         let new_mint = SplTokenPubkey::new(&[5; 32]);
-        *account = TokenAccount {
-            mint: new_mint,
-            owner,
-            delegate: COption::Some(delegate),
-            amount: 42,
-            is_initialized: true,
-            is_native: false,
-            delegated_amount: 30,
-        };
+        TokenAccount::unpack_unchecked_mut(&mut account_data, &mut |account: &mut TokenAccount| {
+            *account = TokenAccount {
+                mint: new_mint,
+                owner,
+                delegate: COption::Some(delegate),
+                amount: 42,
+                state: TokenAccountState::Initialized,
+                is_native: COption::None,
+                delegated_amount: 30,
+                close_authority: COption::Some(owner),
+            };
+            Ok(())
+        })
+        .unwrap();
         let token_account = Account {
             lamports: 111,
             data: account_data.to_vec(),
-            owner: spl_token_id_v1_0(),
+            owner: spl_token_id_v2_0(),
             ..Account::default()
         };
         let token_with_different_mint_pubkey = Pubkey::new_rand();
@@ -4882,7 +4883,7 @@ pub mod tests {
                 "params":["{}", {{"programId": "{}"}}]
             }}"#,
             owner,
-            spl_token_id_v1_0(),
+            spl_token_id_v2_0(),
         );
         let res = io.handle_request_sync(&req, meta.clone());
         let result: Value = serde_json::from_str(&res.expect("actual response"))
@@ -4891,6 +4892,24 @@ pub mod tests {
             serde_json::from_value(result["result"]["value"].clone()).unwrap();
         assert_eq!(accounts.len(), 3);
 
+        // Test getTokenAccountsByOwner with jsonParsed encoding doesn't return accounts with invalid mints
+        let req = format!(
+            r#"{{
+                "jsonrpc":"2.0",
+                "id":1,
+                "method":"getTokenAccountsByOwner",
+                "params":["{}", {{"programId": "{}"}}, {{"encoding": "jsonParsed"}}]
+            }}"#,
+            owner,
+            spl_token_id_v2_0(),
+        );
+        let res = io.handle_request_sync(&req, meta.clone());
+        let result: Value = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        let accounts: Vec<RpcKeyedAccount> =
+            serde_json::from_value(result["result"]["value"].clone()).unwrap();
+        assert_eq!(accounts.len(), 2);
+
         // Test returns only mint accounts
         let req = format!(
             r#"{{
@@ -4946,7 +4965,7 @@ pub mod tests {
                 "params":["{}", {{"programId": "{}"}}]
             }}"#,
             Pubkey::new_rand(),
-            spl_token_id_v1_0(),
+            spl_token_id_v2_0(),
         );
         let res = io.handle_request_sync(&req, meta.clone());
         let result: Value = serde_json::from_str(&res.expect("actual response"))
@@ -4964,7 +4983,7 @@ pub mod tests {
                 "params":["{}", {{"programId": "{}"}}]
             }}"#,
             delegate,
-            spl_token_id_v1_0(),
+            spl_token_id_v2_0(),
         );
         let res = io.handle_request_sync(&req, meta.clone());
         let result: Value = serde_json::from_str(&res.expect("actual response"))
@@ -5029,7 +5048,7 @@ pub mod tests {
                 "params":["{}", {{"programId": "{}"}}]
             }}"#,
             Pubkey::new_rand(),
-            spl_token_id_v1_0(),
+            spl_token_id_v2_0(),
         );
         let res = io.handle_request_sync(&req, meta.clone());
         let result: Value = serde_json::from_str(&res.expect("actual response"))
@@ -5039,40 +5058,47 @@ pub mod tests {
         assert!(accounts.is_empty());
 
         // Add new_mint, and another token account on new_mint with different balance
-        let mut mint_data = [0; size_of::<Mint>()];
-        let mint_state: &mut Mint =
-            spl_token_v1_0::state::unpack_unchecked(&mut mint_data).unwrap();
-        *mint_state = Mint {
-            owner: COption::Some(owner),
-            decimals: 2,
-            is_initialized: true,
-        };
+        let mut mint_data = vec![0; Mint::get_packed_len()];
+        Mint::unpack_unchecked_mut(&mut mint_data, &mut |mint: &mut Mint| {
+            *mint = Mint {
+                mint_authority: COption::Some(owner),
+                supply: 500,
+                decimals: 2,
+                is_initialized: true,
+                freeze_authority: COption::Some(owner),
+            };
+            Ok(())
+        })
+        .unwrap();
         let mint_account = Account {
             lamports: 111,
             data: mint_data.to_vec(),
-            owner: spl_token_id_v1_0(),
+            owner: spl_token_id_v2_0(),
             ..Account::default()
         };
         bank.store_account(
             &Pubkey::from_str(&new_mint.to_string()).unwrap(),
             &mint_account,
         );
-        let mut account_data = [0; size_of::<TokenAccount>()];
-        let account: &mut TokenAccount =
-            spl_token_v1_0::state::unpack_unchecked(&mut account_data).unwrap();
-        *account = TokenAccount {
-            mint: new_mint,
-            owner,
-            delegate: COption::Some(delegate),
-            amount: 10,
-            is_initialized: true,
-            is_native: false,
-            delegated_amount: 30,
-        };
+        let mut account_data = vec![0; TokenAccount::get_packed_len()];
+        TokenAccount::unpack_unchecked_mut(&mut account_data, &mut |account: &mut TokenAccount| {
+            *account = TokenAccount {
+                mint: new_mint,
+                owner,
+                delegate: COption::Some(delegate),
+                amount: 10,
+                state: TokenAccountState::Initialized,
+                is_native: COption::None,
+                delegated_amount: 30,
+                close_authority: COption::Some(owner),
+            };
+            Ok(())
+        })
+        .unwrap();
         let token_account = Account {
             lamports: 111,
             data: account_data.to_vec(),
-            owner: spl_token_id_v1_0(),
+            owner: spl_token_id_v2_0(),
             ..Account::default()
         };
         let token_with_smaller_balance = Pubkey::new_rand();
@@ -5115,43 +5141,50 @@ pub mod tests {
     fn test_token_parsing() {
         let RpcHandler { io, meta, bank, .. } = start_rpc_handler_with_tx(&Pubkey::new_rand());
 
-        let mut account_data = [0; size_of::<TokenAccount>()];
-        let account: &mut TokenAccount =
-            spl_token_v1_0::state::unpack_unchecked(&mut account_data).unwrap();
+        let mut account_data = vec![0; TokenAccount::get_packed_len()];
         let mint = SplTokenPubkey::new(&[2; 32]);
         let owner = SplTokenPubkey::new(&[3; 32]);
         let delegate = SplTokenPubkey::new(&[4; 32]);
-        *account = TokenAccount {
-            mint,
-            owner,
-            delegate: COption::Some(delegate),
-            amount: 420,
-            is_initialized: true,
-            is_native: false,
-            delegated_amount: 30,
-        };
+        TokenAccount::unpack_unchecked_mut(&mut account_data, &mut |account: &mut TokenAccount| {
+            *account = TokenAccount {
+                mint,
+                owner,
+                delegate: COption::Some(delegate),
+                amount: 420,
+                state: TokenAccountState::Initialized,
+                is_native: COption::Some(10),
+                delegated_amount: 30,
+                close_authority: COption::Some(owner),
+            };
+            Ok(())
+        })
+        .unwrap();
         let token_account = Account {
             lamports: 111,
             data: account_data.to_vec(),
-            owner: spl_token_id_v1_0(),
+            owner: spl_token_id_v2_0(),
             ..Account::default()
         };
         let token_account_pubkey = Pubkey::new_rand();
         bank.store_account(&token_account_pubkey, &token_account);
 
         // Add the mint
-        let mut mint_data = [0; size_of::<Mint>()];
-        let mint_state: &mut Mint =
-            spl_token_v1_0::state::unpack_unchecked(&mut mint_data).unwrap();
-        *mint_state = Mint {
-            owner: COption::Some(owner),
-            decimals: 2,
-            is_initialized: true,
-        };
+        let mut mint_data = vec![0; Mint::get_packed_len()];
+        Mint::unpack_unchecked_mut(&mut mint_data, &mut |mint: &mut Mint| {
+            *mint = Mint {
+                mint_authority: COption::Some(owner),
+                supply: 500,
+                decimals: 2,
+                is_initialized: true,
+                freeze_authority: COption::Some(owner),
+            };
+            Ok(())
+        })
+        .unwrap();
         let mint_account = Account {
             lamports: 111,
             data: mint_data.to_vec(),
-            owner: spl_token_id_v1_0(),
+            owner: spl_token_id_v2_0(),
             ..Account::default()
         };
         bank.store_account(&Pubkey::from_str(&mint.to_string()).unwrap(), &mint_account);
@@ -5167,7 +5200,7 @@ pub mod tests {
             result["result"]["value"]["data"],
             json!({
                 "program": "spl-token",
-                "space": 120,
+                "space": TokenAccount::get_packed_len(),
                 "parsed": {
                     "type": "account",
                     "info": {
@@ -5179,13 +5212,19 @@ pub mod tests {
                             "amount": "420",
                         },
                         "delegate": delegate.to_string(),
-                        "isInitialized": true,
-                        "isNative": false,
+                        "state": "initialized",
+                        "isNative": true,
+                        "rentExemptReserve": {
+                            "uiAmount": 0.1,
+                            "decimals": 2,
+                            "amount": "10",
+                        },
                         "delegatedAmount": {
                             "uiAmount": 0.3,
                             "decimals": 2,
                             "amount": "30",
                         },
+                        "closeAuthority": owner.to_string(),
                     }
                 }
             })
@@ -5203,13 +5242,15 @@ pub mod tests {
             result["result"]["value"]["data"],
             json!({
                 "program": "spl-token",
-                "space": 40,
+                "space": Mint::get_packed_len(),
                 "parsed": {
                     "type": "mint",
                     "info": {
-                        "owner": owner.to_string(),
+                        "mintAuthority": owner.to_string(),
                         "decimals": 2,
+                        "supply": "500".to_string(),
                         "isInitialized": true,
+                        "freezeAuthority": owner.to_string(),
                     }
                 }
             })

core/src/rpc_error.rs

@@ -4,6 +4,7 @@ use solana_sdk::clock::Slot;
 const JSON_RPC_SERVER_ERROR_1: i64 = -32001;
 const JSON_RPC_SERVER_ERROR_2: i64 = -32002;
 const JSON_RPC_SERVER_ERROR_3: i64 = -32003;
+const JSON_RPC_SERVER_ERROR_4: i64 = -32004;
 
 pub enum RpcCustomError {
     BlockCleanedUp {
@@ -14,6 +15,9 @@ pub enum RpcCustomError {
         message: String,
     },
     SendTransactionIsNotSigned,
+    BlockNotAvailable {
+        slot: Slot,
+    },
 }
 
 impl From<RpcCustomError> for Error {
@@ -40,6 +44,11 @@ impl From<RpcCustomError> for Error {
                 message: "Transaction is not signed".to_string(),
                 data: None,
             },
+            RpcCustomError::BlockNotAvailable { slot } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_4),
+                message: format!("Block not available for slot {}", slot,),
+                data: None,
+            },
         }
     }
 }

core/src/rpc_subscriptions.rs

@@ -8,7 +8,7 @@ use jsonrpc_pubsub::{
     SubscriptionId,
 };
 use serde::Serialize;
-use solana_account_decoder::{parse_token::spl_token_id_v1_0, UiAccount, UiAccountEncoding};
+use solana_account_decoder::{parse_token::spl_token_id_v2_0, UiAccount, UiAccountEncoding};
 use solana_client::{
     rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig},
     rpc_filter::RpcFilterType,
@@ -256,7 +256,7 @@ fn filter_account_result(
         // and should notify that the account state has been reverted.
         if fork != last_notified_slot {
             let encoding = encoding.unwrap_or(UiAccountEncoding::Binary);
-            if account.owner == spl_token_id_v1_0() && encoding == UiAccountEncoding::JsonParsed {
+            if account.owner == spl_token_id_v2_0() && encoding == UiAccountEncoding::JsonParsed {
                 let bank = bank.unwrap(); // If result.is_some(), bank must also be Some
                 return (
                     Box::new(iter::once(get_parsed_token_account(bank, pubkey, account))),

core/src/serve_repair.rs

@@ -21,7 +21,7 @@ use solana_sdk::{
 };
 use solana_streamer::streamer::{PacketReceiver, PacketSender};
 use std::{
-    collections::HashMap,
+    collections::{HashMap, HashSet},
     net::SocketAddr,
     sync::atomic::{AtomicBool, Ordering},
     sync::{Arc, RwLock},
@@ -382,12 +382,13 @@ impl ServeRepair {
         repair_request: RepairType,
         cache: &mut RepairCache,
         repair_stats: &mut RepairStats,
+        repair_validators: &Option<HashSet<Pubkey>>,
     ) -> Result<(SocketAddr, Vec<u8>)> {
         // find a peer that appears to be accepting replication and has the desired slot, as indicated
         // by a valid tvu port location
         let slot = repair_request.slot();
         if cache.get(&slot).is_none() {
-            let repair_peers: Vec<_> = self.cluster_info.repair_peers(slot);
+            let repair_peers = self.repair_peers(&repair_validators, slot);
             if repair_peers.is_empty() {
                 return Err(ClusterInfoError::NoPeers.into());
             }
@@ -411,8 +412,9 @@ impl ServeRepair {
         &self,
         slot: Slot,
         cluster_slots: &ClusterSlots,
+        repair_validators: &Option<HashSet<Pubkey>>,
     ) -> Result<(Pubkey, SocketAddr)> {
-        let repair_peers: Vec<_> = self.cluster_info.repair_peers(slot);
+        let repair_peers: Vec<_> = self.repair_peers(repair_validators, slot);
         if repair_peers.is_empty() {
             return Err(ClusterInfoError::NoPeers.into());
         }
@@ -448,6 +450,27 @@ impl ServeRepair {
         }
     }
 
+    fn repair_peers(
+        &self,
+        repair_validators: &Option<HashSet<Pubkey>>,
+        slot: Slot,
+    ) -> Vec<ContactInfo> {
+        if let Some(repair_validators) = repair_validators {
+            repair_validators
+                .iter()
+                .filter_map(|key| {
+                    if *key != self.my_info.id {
+                        self.cluster_info.lookup_contact_info(key, |ci| ci.clone())
+                    } else {
+                        None
+                    }
+                })
+                .collect()
+        } else {
+            self.cluster_info.repair_peers(slot)
+        }
+    }
+
     fn run_window_request(
         recycler: &PacketsRecycler,
         from: &ContactInfo,
@@ -662,7 +685,7 @@ mod tests {
                 repair: socketaddr!("127.0.0.1:1237"),
                 tpu: socketaddr!("127.0.0.1:1238"),
                 tpu_forwards: socketaddr!("127.0.0.1:1239"),
-                unused: socketaddr!("127.0.0.1:1240"),
+                rpc_banks: socketaddr!("127.0.0.1:1240"),
                 rpc: socketaddr!("127.0.0.1:1241"),
                 rpc_pubsub: socketaddr!("127.0.0.1:1242"),
                 serve_repair: socketaddr!("127.0.0.1:1243"),
@@ -733,6 +756,7 @@ mod tests {
             RepairType::Shred(0, 0),
             &mut HashMap::new(),
             &mut RepairStats::default(),
+            &None,
         );
         assert_matches!(rv, Err(Error::ClusterInfoError(ClusterInfoError::NoPeers)));
 
@@ -745,7 +769,7 @@ mod tests {
             repair: socketaddr!([127, 0, 0, 1], 1237),
             tpu: socketaddr!([127, 0, 0, 1], 1238),
             tpu_forwards: socketaddr!([127, 0, 0, 1], 1239),
-            unused: socketaddr!([127, 0, 0, 1], 1240),
+            rpc_banks: socketaddr!([127, 0, 0, 1], 1240),
             rpc: socketaddr!([127, 0, 0, 1], 1241),
             rpc_pubsub: socketaddr!([127, 0, 0, 1], 1242),
             serve_repair: serve_repair_addr,
@@ -759,6 +783,7 @@ mod tests {
                 RepairType::Shred(0, 0),
                 &mut HashMap::new(),
                 &mut RepairStats::default(),
+                &None,
             )
             .unwrap();
         assert_eq!(nxt.serve_repair, serve_repair_addr);
@@ -773,7 +798,7 @@ mod tests {
             repair: socketaddr!([127, 0, 0, 1], 1237),
             tpu: socketaddr!([127, 0, 0, 1], 1238),
             tpu_forwards: socketaddr!([127, 0, 0, 1], 1239),
-            unused: socketaddr!([127, 0, 0, 1], 1240),
+            rpc_banks: socketaddr!([127, 0, 0, 1], 1240),
             rpc: socketaddr!([127, 0, 0, 1], 1241),
             rpc_pubsub: socketaddr!([127, 0, 0, 1], 1242),
             serve_repair: serve_repair_addr2,
@@ -791,6 +816,7 @@ mod tests {
                     RepairType::Shred(0, 0),
                     &mut HashMap::new(),
                     &mut RepairStats::default(),
+                    &None,
                 )
                 .unwrap();
             if rv.0 == serve_repair_addr {
@@ -937,4 +963,71 @@ mod tests {
 
         Blockstore::destroy(&ledger_path).expect("Expected successful database destruction");
     }
+
+    #[test]
+    fn test_repair_with_repair_validators() {
+        let cluster_slots = ClusterSlots::default();
+        let me = ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp());
+        let cluster_info = Arc::new(ClusterInfo::new_with_invalid_keypair(me.clone()));
+
+        // Insert two peers on the network
+        let contact_info2 = ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp());
+        let contact_info3 = ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp());
+        cluster_info.insert_info(contact_info2.clone());
+        cluster_info.insert_info(contact_info3.clone());
+        let serve_repair = ServeRepair::new(cluster_info);
+
+        // If:
+        // 1) repair validator set doesn't exist in gossip
+        // 2) repair validator set only includes our own id
+        // then no repairs should be generated
+        for pubkey in &[Pubkey::new_rand(), me.id] {
+            let trusted_validators = Some(vec![*pubkey].into_iter().collect());
+            assert!(serve_repair.repair_peers(&trusted_validators, 1).is_empty());
+            assert!(serve_repair
+                .repair_request(
+                    &cluster_slots,
+                    RepairType::Shred(0, 0),
+                    &mut HashMap::new(),
+                    &mut RepairStats::default(),
+                    &trusted_validators,
+                )
+                .is_err());
+        }
+
+        // If trusted validator exists in gossip, should return repair successfully
+        let trusted_validators = Some(vec![contact_info2.id].into_iter().collect());
+        let repair_peers = serve_repair.repair_peers(&trusted_validators, 1);
+        assert_eq!(repair_peers.len(), 1);
+        assert_eq!(repair_peers[0].id, contact_info2.id);
+        assert!(serve_repair
+            .repair_request(
+                &cluster_slots,
+                RepairType::Shred(0, 0),
+                &mut HashMap::new(),
+                &mut RepairStats::default(),
+                &trusted_validators,
+            )
+            .is_ok());
+
+        // Using no trusted validators should default to all
+        // validator's available in gossip, excluding myself
+        let repair_peers: HashSet<Pubkey> = serve_repair
+            .repair_peers(&None, 1)
+            .into_iter()
+            .map(|c| c.id)
+            .collect();
+        assert_eq!(repair_peers.len(), 2);
+        assert!(repair_peers.contains(&contact_info2.id));
+        assert!(repair_peers.contains(&contact_info3.id));
+        assert!(serve_repair
+            .repair_request(
+                &cluster_slots,
+                RepairType::Shred(0, 0),
+                &mut HashMap::new(),
+                &mut RepairStats::default(),
+                &None,
+            )
+            .is_ok());
+    }
 }

core/src/tvu.rs

@@ -66,6 +66,7 @@ pub struct TvuConfig {
     pub shred_version: u16,
     pub halt_on_trusted_validators_accounts_hash_mismatch: bool,
     pub trusted_validators: Option<HashSet<Pubkey>>,
+    pub repair_validators: Option<HashSet<Pubkey>>,
     pub accounts_hash_fault_injection_slots: u64,
 }
 
@@ -150,6 +151,7 @@ impl Tvu {
             cluster_slots.clone(),
             duplicate_slots_reset_sender,
             verified_vote_receiver,
+            tvu_config.repair_validators,
         );
 
         let (ledger_cleanup_slot_sender, ledger_cleanup_slot_receiver) = channel();

core/src/validator.rs

@@ -23,6 +23,7 @@ use crate::{
 };
 use crossbeam_channel::unbounded;
 use rand::{thread_rng, Rng};
+use solana_banks_server::rpc_banks_service::RpcBanksService;
 use solana_ledger::{
     bank_forks_utils,
     blockstore::{Blockstore, CompletedSlotsReceiver, PurgeType},
@@ -72,7 +73,7 @@ pub struct ValidatorConfig {
     pub voting_disabled: bool,
     pub account_paths: Vec<PathBuf>,
     pub rpc_config: JsonRpcConfig,
-    pub rpc_ports: Option<(u16, u16)>, // (API, PubSub)
+    pub rpc_ports: Option<(u16, u16, u16)>, // (JsonRpc, JsonRpcPubSub, Banks)
     pub snapshot_config: Option<SnapshotConfig>,
     pub max_ledger_shreds: Option<u64>,
     pub broadcast_stage_type: BroadcastStageType,
@@ -81,6 +82,7 @@ pub struct ValidatorConfig {
     pub wait_for_supermajority: Option<Slot>,
     pub new_hard_forks: Option<Vec<Slot>>,
     pub trusted_validators: Option<HashSet<Pubkey>>, // None = trust all
+    pub repair_validators: Option<HashSet<Pubkey>>,  // None = repair from all
     pub halt_on_trusted_validators_accounts_hash_mismatch: bool,
     pub accounts_hash_fault_injection_slots: u64, // 0 = no fault injection
     pub frozen_accounts: Vec<Pubkey>,
@@ -109,6 +111,7 @@ impl Default for ValidatorConfig {
             wait_for_supermajority: None,
             new_hard_forks: None,
             trusted_validators: None,
+            repair_validators: None,
             halt_on_trusted_validators_accounts_hash_mismatch: false,
             accounts_hash_fault_injection_slots: 0,
             frozen_accounts: vec![],
@@ -148,7 +151,7 @@ struct TransactionHistoryServices {
 pub struct Validator {
     pub id: Pubkey,
     validator_exit: Arc<RwLock<Option<ValidatorExit>>>,
-    rpc_service: Option<(JsonRpcService, PubSubService)>,
+    rpc_service: Option<(JsonRpcService, PubSubService, RpcBanksService)>,
     transaction_status_service: Option<TransactionStatusService>,
     rewards_recorder_service: Option<RewardsRecorderService>,
     gossip_service: GossipService,
@@ -282,36 +285,47 @@ impl Validator {
         ));
 
         let rpc_override_health_check = Arc::new(AtomicBool::new(false));
-        let rpc_service = config.rpc_ports.map(|(rpc_port, rpc_pubsub_port)| {
-            if ContactInfo::is_valid_address(&node.info.rpc) {
-                assert!(ContactInfo::is_valid_address(&node.info.rpc_pubsub));
-                assert_eq!(rpc_port, node.info.rpc.port());
-                assert_eq!(rpc_pubsub_port, node.info.rpc_pubsub.port());
-            } else {
-                assert!(!ContactInfo::is_valid_address(&node.info.rpc_pubsub));
-            }
-            (
-                JsonRpcService::new(
-                    SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), rpc_port),
-                    config.rpc_config.clone(),
-                    config.snapshot_config.clone(),
-                    bank_forks.clone(),
-                    block_commitment_cache.clone(),
-                    blockstore.clone(),
-                    cluster_info.clone(),
-                    genesis_config.hash(),
-                    ledger_path,
-                    validator_exit.clone(),
-                    config.trusted_validators.clone(),
-                    rpc_override_health_check.clone(),
-                ),
-                PubSubService::new(
-                    &subscriptions,
-                    SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), rpc_pubsub_port),
-                    &exit,
-                ),
-            )
-        });
+        let rpc_service = config
+            .rpc_ports
+            .map(|(rpc_port, rpc_pubsub_port, rpc_banks_port)| {
+                if ContactInfo::is_valid_address(&node.info.rpc) {
+                    assert!(ContactInfo::is_valid_address(&node.info.rpc_pubsub));
+                    assert_eq!(rpc_port, node.info.rpc.port());
+                    assert_eq!(rpc_pubsub_port, node.info.rpc_pubsub.port());
+                    assert_eq!(rpc_banks_port, node.info.rpc_banks.port());
+                } else {
+                    assert!(!ContactInfo::is_valid_address(&node.info.rpc_pubsub));
+                }
+                let tpu_address = cluster_info.my_contact_info().tpu;
+                (
+                    JsonRpcService::new(
+                        SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), rpc_port),
+                        config.rpc_config.clone(),
+                        config.snapshot_config.clone(),
+                        bank_forks.clone(),
+                        block_commitment_cache.clone(),
+                        blockstore.clone(),
+                        cluster_info.clone(),
+                        genesis_config.hash(),
+                        ledger_path,
+                        validator_exit.clone(),
+                        config.trusted_validators.clone(),
+                        rpc_override_health_check.clone(),
+                    ),
+                    PubSubService::new(
+                        &subscriptions,
+                        SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), rpc_pubsub_port),
+                        &exit,
+                    ),
+                    RpcBanksService::new(
+                        SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), rpc_banks_port),
+                        tpu_address,
+                        &bank_forks,
+                        &block_commitment_cache,
+                        &exit,
+                    ),
+                )
+            });
 
         info!(
             "Starting PoH: epoch={} slot={} tick_height={} blockhash={} leader={:?}",
@@ -460,6 +474,7 @@ impl Validator {
                     .halt_on_trusted_validators_accounts_hash_mismatch,
                 shred_version: node.info.shred_version,
                 trusted_validators: config.trusted_validators.clone(),
+                repair_validators: config.repair_validators.clone(),
                 accounts_hash_fault_injection_slots: config.accounts_hash_fault_injection_slots,
             },
         );
@@ -543,9 +558,10 @@ impl Validator {
     pub fn join(self) -> Result<()> {
         self.poh_service.join()?;
         drop(self.poh_recorder);
-        if let Some((rpc_service, rpc_pubsub_service)) = self.rpc_service {
+        if let Some((rpc_service, rpc_pubsub_service, rpc_banks_service)) = self.rpc_service {
             rpc_service.join()?;
             rpc_pubsub_service.join()?;
+            rpc_banks_service.join()?;
         }
         if let Some(transaction_status_service) = self.transaction_status_service {
             transaction_status_service.join()?;
@@ -850,18 +866,14 @@ impl TestValidator {
         } = create_genesis_config_with_leader_ex(
             mint_lamports,
             &contact_info.id,
-            Arc::new(Keypair::new()),
-            Arc::new(Keypair::new()),
+            &Keypair::new(),
+            &Pubkey::new_rand(),
             42,
             bootstrap_validator_lamports,
         );
         genesis_config
             .native_instruction_processors
             .push(solana_budget_program!());
-        genesis_config
-            .native_instruction_processors
-            .push(solana_bpf_loader_program!());
-
         genesis_config.rent.lamports_per_byte_year = 1;
         genesis_config.rent.exemption_threshold = 1.0;
         genesis_config.fee_rate_governor = FeeRateGovernor::new(fees, 0);
@@ -869,15 +881,20 @@ impl TestValidator {
         let (ledger_path, blockhash) = create_new_tmp_ledger!(&genesis_config);
 
         let config = ValidatorConfig {
-            rpc_ports: Some((node.info.rpc.port(), node.info.rpc_pubsub.port())),
+            rpc_ports: Some((
+                node.info.rpc.port(),
+                node.info.rpc_pubsub.port(),
+                node.info.rpc_banks.port(),
+            )),
             ..ValidatorConfig::default()
         };
+        let vote_pubkey = voting_keypair.pubkey();
         let node = Validator::new(
             node,
             &node_keypair,
             &ledger_path,
             &voting_keypair.pubkey(),
-            vec![voting_keypair.clone()],
+            vec![Arc::new(voting_keypair)],
             None,
             true,
             &config,
@@ -889,7 +906,7 @@ impl TestValidator {
             alice: mint_keypair,
             ledger_path,
             genesis_hash: blockhash,
-            vote_pubkey: voting_keypair.pubkey(),
+            vote_pubkey,
         }
     }
 }
@@ -1037,6 +1054,7 @@ mod tests {
             rpc_ports: Some((
                 validator_node.info.rpc.port(),
                 validator_node.info.rpc_pubsub.port(),
+                validator_node.info.rpc_banks.port(),
             )),
             ..ValidatorConfig::default()
         };
@@ -1106,11 +1124,12 @@ mod tests {
                         .genesis_config;
                 let (validator_ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config);
                 ledger_paths.push(validator_ledger_path.clone());
-                let vote_account_keypair = Arc::new(Keypair::new());
+                let vote_account_keypair = Keypair::new();
                 let config = ValidatorConfig {
                     rpc_ports: Some((
                         validator_node.info.rpc.port(),
                         validator_node.info.rpc_pubsub.port(),
+                        validator_node.info.rpc_banks.port(),
                     )),
                     ..ValidatorConfig::default()
                 };
@@ -1119,7 +1138,7 @@ mod tests {
                     &Arc::new(validator_keypair),
                     &validator_ledger_path,
                     &vote_account_keypair.pubkey(),
-                    vec![vote_account_keypair.clone()],
+                    vec![Arc::new(vote_account_keypair)],
                     Some(&leader_node.info),
                     true,
                     &config,

core/tests/gossip.rs

@@ -233,7 +233,9 @@ pub fn cluster_info_scale() {
 
     let nodes: Vec<_> = vote_keypairs
         .into_iter()
-        .map(|keypairs| test_node_with_bank(keypairs.node_keypair, &exit, bank_forks.clone()))
+        .map(|keypairs| {
+            test_node_with_bank(Arc::new(keypairs.node_keypair), &exit, bank_forks.clone())
+        })
         .collect();
     let ci0 = nodes[0].0.my_contact_info();
     for node in &nodes[1..] {

crate-features/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-crate-features"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana Crate Features"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

docs/sidebars.js

@@ -16,6 +16,14 @@ module.exports = {
           "wallet-guide/ledger-live",
         ],
       },
+      {
+        type: "category",
+        label: "Web Wallets",
+        items: [
+          "wallet-guide/web-wallets",
+          "wallet-guide/solflare",
+        ],
+      },
       {
         type: "category",
         label: "Command-line Wallets",

docs/src/apps/jsonrpc-api.md

@@ -241,7 +241,7 @@ curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "m
 
 ### getBlockTime
 
-Returns the estimated production time of a block.
+Returns the estimated production time of a confirmed block.
 
 Each validator reports their UTC time to the ledger on a regular interval by
 intermittently adding a timestamp to a Vote for a particular block. A requested
@@ -259,8 +259,8 @@ query a node that is built from genesis and retains the entire ledger.
 
 #### Results:
 
-- `<null>` - block has not yet been produced
-- `<i64>` - estimated production time, as Unix timestamp (seconds since the Unix epoch)
+* `<i64>` - estimated production time, as Unix timestamp (seconds since the Unix epoch)
+* `<null>` - timestamp is not available for this block
 
 #### Example:
 
@@ -846,7 +846,7 @@ Returns all accounts owned by the provided program Pubkey
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
   - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64" or jsonParsed".
-    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to base64 encoding, detectable when the `data` field is type `<string>`. **jsonParsed encoding is UNSTABLE**
+    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to base64 encoding, detectable when the `data` field is type `<string>`. If parsed-JSON is requested for the SPL Token program, when a valid mint cannot be found for a particular account, that account will be filtered out from results. **jsonParsed encoding is UNSTABLE**
   - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "base58" or "base64" encoding.
   - (optional) `filters: <array>` - filter results using various [filter objects](jsonrpc-api.md#filters); account must meet all filter criteria to be included in results
 
@@ -1101,7 +1101,7 @@ Returns all SPL Token accounts by approved Delegate. **UNSTABLE**
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
   - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64" or jsonParsed".
-    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`. **jsonParsed encoding is UNSTABLE**
+    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a valid mint cannot be found for a particular account, that account will be filtered out from results. **jsonParsed encoding is UNSTABLE**
   - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "base58" or "base64" encoding.
 
 #### Results:
@@ -1138,7 +1138,7 @@ Returns all SPL Token accounts by token owner. **UNSTABLE**
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
   - `encoding: <string>` - encoding for Account data, either "base58" (*slow*), "base64" or jsonParsed".
-    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`. **jsonParsed encoding is UNSTABLE**
+    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a valid mint cannot be found for a particular account, that account will be filtered out from results. **jsonParsed encoding is UNSTABLE**
   - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "base58" or "base64" encoding.
 
 #### Results:
@@ -1257,7 +1257,7 @@ The result field will be a JSON object with the following fields:
 // Request
 curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getVersion"}' http://localhost:8899
 // Result
-{"jsonrpc":"2.0","result":{"solana-core": "1.3.4"},"id":1}
+{"jsonrpc":"2.0","result":{"solana-core": "1.3.6"},"id":1}
 ```
 
 ### getVoteAccounts
@@ -1353,6 +1353,7 @@ Before submitting, the following preflight checks are performed:
 - `<string>` - fully-signed Transaction, as base-58 encoded string
 - `<object>` - (optional) Configuration object containing the following field:
   - `skipPreflight: <bool>` - if true, skip the preflight transaction checks (default: false)
+  - `preflightCommitment: <object>` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) level to use for preflight (default: max).
 
 #### Results:
 
@@ -1377,6 +1378,7 @@ Simulate sending a transaction
 - `<string>` - Transaction, as base-58 encoded string. The transaction must have a valid blockhash, but is not required to be signed.
 - `<object>` - (optional) Configuration object containing the following field:
   - `sigVerify: <bool>` - if true the transaction signatures will be verified (default: false)
+  - `commitment: <object>` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) level to simulate the transaction at (default: max).
 
 #### Results:
 

docs/src/clusters.md

@@ -39,14 +39,14 @@ solana config set --url https://devnet.solana.com
 $ solana-validator \
     --identity ~/validator-keypair.json \
     --vote-account ~/vote-account-keypair.json \
-    --trusted-validator 47Sbuv6jL7CViK9F2NMW51aQGhfdpUu7WNvKyH645Rfi \
+    --trusted-validator dv1LfzJvDF7S1fBKpFgKoKXK5yoSosmkAdfbxBo1GqJ \
     --no-untrusted-rpc \
     --ledger ~/validator-ledger \
     --rpc-port 8899 \
     --dynamic-port-range 8000-8010 \
-    --entrypoint devnet.solana.com:8001 \
-    --expected-genesis-hash HzyuivuNXMHJKjM6q6BE2qBsR3etqW21BSvuJTpJFj9A \
-    --expected-shred-version 61357 \
+    --entrypoint 35.197.53.105:8001 \
+    --expected-genesis-hash Ap36zrBt2jLWpwUjaF48hRULVgmvSE3ViFxiQgjZX2XC \
+    --expected-shred-version 37460 \
     --limit-ledger-size
 ```
 
@@ -81,7 +81,7 @@ $ solana-validator \
     --identity ~/validator-keypair.json \
     --vote-account ~/vote-account-keypair.json \
     --trusted-validator 5D1fNXzvv5NjV1ysLjirC4WY92RNsVH18vjmcszZd8on \
-    --trusted-validator Ft5fbkqNa76vnsjYNwjDZUXoTWpP7VYm3mtsaQckQAD \
+    --trusted-validator Ft5fbkqNa76vnsjYNwjDZUXoTWpP7VYm3mtsaQckQADN \
     --trusted-validator 9QxCLckBiJc783jnMvXZubK4wH86Eqqvashtrwvcsgkv \
     --no-untrusted-rpc \
     --ledger ~/validator-ledger \

docs/src/integrations/exchange.md

@@ -100,6 +100,77 @@ greater security. If so, you will need to move SOL to hot accounts using our
 When a user wants to deposit SOL into your exchange, instruct them to send a
 transfer to the appropriate deposit address.
 
+## Validating User-supplied Account Addresses for Withdrawals in SOL
+
+As withdrawals are irreversible, it may be a good practice to validate the
+account address before authorizing withdrawals into user-supplied accounts
+to prevent accidental user's fund loss.
+
+For a normal account in Solana, its address is simply a Base58-encoded
+actual 256-bit public key of ed25519. Because not all bit pattern is a valid
+public key for the ed25519, it's possible to make sure user-supplied
+account addresses are at least something that may be a correct ed25519 public
+key.
+
+### Java
+
+You can check Solana's normal account address validity by first decoding
+Base58 string and ensuring the decoded bytes are valid ed25519 public keys
+like this:
+
+The following code sample assumes you're using the Maven.
+
+`pom.xml`:
+
+```xml
+<repositories>
+  ...
+  <repository>
+    <id>spring</id>
+    <url>https://repo.spring.io/libs-release/</url>
+  </repository>
+</repositories>
+
+...
+
+<dependencies>
+  ...
+  <dependency>
+      <groupId>io.github.novacrypto</groupId>
+      <artifactId>Base58</artifactId>
+      <version>0.1.3</version>
+  </dependency>
+  <dependency>
+      <groupId>cafe.cryptography</groupId>
+      <artifactId>curve25519-elisabeth</artifactId>
+      <version>0.1.0</version>
+  </dependency>
+<dependencies>
+```
+
+```java
+import io.github.novacrypto.base58.Base58;
+import cafe.cryptography.curve25519.CompressedEdwardsY;
+
+public class PubkeyValidator
+{
+    public static boolean verifyPubkey(String userProvidedPubkey)
+    {
+        try {
+            return _verifyPubkeyInternal(userProvidedPubkey);
+        } catch (Exception e) {
+            return false;
+        }
+    }
+
+    public static boolean _verifyPubkeyInternal(String maybePubkey) throws Exception
+    {
+        byte[] bytes = Base58.base58Decode(maybePubkey);
+        return !(new CompressedEdwardsY(bytes)).decompress().isSmallOrder();
+    }
+}
+```
+
 ### Poll for Blocks
 
 The easiest way to track all the deposit accounts for your exchange is to poll

docs/src/running-validator/validator-start.md

@@ -46,10 +46,14 @@ that CUDA is enabled: `"[<timestamp> solana::validator] CUDA is enabled"`
 
 ## System Tuning
 
-For Linux validators, the solana repo includes a daemon to adjust system settings to optimize
-performance (namely by increasing the OS UDP buffer limits, and scheduling PoH with realtime policy).
+### Linux
+#### Automatic
+The solana repo includes a daemon to adjust system settings to optimize performance
+(namely by increasing the OS UDP buffer limits, and scheduling PoH with realtime policy).
 
-The daemon (`solana-sys-tuner`) is included in the solana binary release.
+The daemon (`solana-sys-tuner`) is included in the solana binary release. Restart
+it, *before* restarting your validator, after each software upgrade to ensure that
+the latest recommended settings are applied.
 
 To run it:
 
@@ -57,6 +61,53 @@ To run it:
 sudo solana-sys-tuner --user $(whoami) > sys-tuner.log 2>&1 &
 ```
 
+#### Manual
+If you would prefer to manage system settings on your own, you may do so with
+the following commands.
+
+##### **Increase UDP buffers**
+```bash
+sudo bash -c "cat >/etc/sysctl.d/20-solana-udp-buffers.conf <<EOF
+# Increase UDP buffer size
+net.core.rmem_default = 134217728
+net.core.rmem_max = 134217728
+net.core.wmem_default = 134217728
+net.core.wmem_max = 134217728
+EOF"
+```
+```bash
+sudo sysctl -p /etc/sysctl.d/20-solana-udp-buffers.conf
+```
+
+##### **Increased memory mapped files limit**
+```bash
+sudo bash -c "cat >/etc/sysctl.d/20-solana-mmaps.conf <<EOF
+# Increase memory mapped files limit
+vm.max_map_count = 500000
+EOF"
+```
+```bash
+sudo sysctl -p /etc/sysctl.d/20-solana-mmaps.conf
+```
+Add
+```
+LimitNOFILE=500000
+```
+to the `[Service]` section of your systemd service file, if you use one,
+otherwise add it to `/etc/systemd/system.conf`.
+```bash
+sudo systemctl daemon-reload
+```
+```bash
+sudo bash -c "cat >/etc/security/limits.d/90-solana-nofiles.conf <<EOF
+# Increase process file descriptor count limit
+* - nofile 500000
+EOF"
+```
+```bash
+### Close all open sessions (log out then, in again) ###
+```
+
 ## Generate identity
 
 Create an identity keypair for your validator by running:

docs/src/staking.md

@@ -64,10 +64,15 @@ Different wallets will vary slightly in their process for this but the general
 description is below.
 
 #### Supported Wallets
-Currently, staking operation are only supported by wallets that can interact
-with the Solana command line tools, including Ledger Nano S and paper wallet.
+Staking operations are supported by the following wallet solutions:
 
-[Staking commands using the Solana Command Line Tools](cli/delegate-stake.md)
+ - SolFlare.com in conjunction with a keystore file or a Ledger Nano S.  Check
+ out our [guide to using SolFlare](wallet-guide/solflare.md) for details.
+
+ - Solana command line tools can perform all stake operations in conjunction
+ with a CLI-generated keypair file wallet, a paper wallet, or with a connected
+ Ledger Nano S.
+ [Staking commands using the Solana Command Line Tools](cli/delegate-stake.md).
 
 #### Create a Stake Account
 A stake account is a different type of account from a wallet address

docs/src/wallet-guide.md

@@ -53,8 +53,9 @@ Solana supports several types of wallets in the Solana native
 command-line app as well as wallets from third-parties.
 
 For the majority of users, we recommend using one of the
-[app wallets](wallet-guide/apps.md), which will provide a more familiar user
-experience rather than needing to learn command line tools.
+[app wallets](wallet-guide/apps.md) or a browser-based
+[web wallet](wallet-guide/web-wallets.md), which will provide a more familiar
+user experience rather than needing to learn command line tools.
 
 For advanced users or developers, the [command-line wallets](wallet-guide/cli.md)
 may be more appropriate, as new features on the Solana blockchain will always be

docs/src/wallet-guide/ledger-live.md

@@ -6,14 +6,9 @@ This document describes how to set up a
 [Ledger Nano S hardware wallet](https://shop.ledger.com/products/ledger-nano-s)
 with the [Ledger Live](https://www.ledger.com/ledger-live) software.
 
-**NOTE: While Solana tools are fully integrated with the Ledger Nano S device,
-and the Solana App can be installed on the Nano S using Ledger Live, adding and
-managing wallet accounts currently requires use of our command-line tools.
-Integration with Ledger Live to use Solana wallet accounts on Ledger Live
-will be available in the future.**
-
-Users may [use a Ledger Nano S with the Solana command
-line tools](hardware-wallets/ledger.md).
+Once the setup steps shown below are complete and the Solana app is installed
+on your Nano S device, users have several options of how to
+[use the Nano S to interact with the Solana Network](#interact-with-the-solana-network)
 
 ## Set up a Ledger Nano S
 
@@ -66,11 +61,20 @@ of the Solana App, please upgrade to version v0.2.2 by following these steps.
 
 ![Upgrade complete](/img/ledger-live-latest-version-installed.png)
 
-## Interact with Solana network
+## Interact with the Solana network
 
-- To interact with your Ledger wallet on our live network, please see our
-  instructions on how to
-  [use a Ledger Nano S with the Solana command line tools](hardware-wallets/ledger.md).
+Users can use any of the following options to sign and submit transactions with
+the Ledger Nano S to interact with the Solana network:
+
+- [SolFlare.com](https://solflare.com/) is a non-custodial web wallet built
+specifically for Solana and supports basic transfers and staking operations
+with the Ledger device.
+Check out our guide for [using a Ledger Nano S with SolFlare](solflare.md).
+
+- Developers and advanced users may
+[use a Ledger Nano S with the Solana command line tools](hardware-wallets/ledger.md).
+New wallet features are almost always supported in the native command line tools
+before being supported by third-party wallets.
 
 ## Support
 

docs/src/wallet-guide/solflare.md

@@ -0,0 +1,185 @@
+---
+title: SolFlare Web Wallet
+---
+
+## Introduction
+
+[SolFlare.com](https://solflare.com/) is a community-created web wallet built
+specifically for Solana.
+SolFlare supports sending and receiving native SOL tokens as well as sending and
+receiving SPL Tokens (Solana's ERC-20 equivalent).
+SolFlare also supports staking of SOL tokens.
+
+As a _non-custodial_ wallet, your private keys are not stored by the SolFlare
+site itself, but rather they are stored in an encrypted
+[Keystore File](#using-a-keystore-file) or on a
+[Ledger Nano S hardware wallet](#using-a-ledger-nano-s-hardware-wallet).
+
+This guide describes how to set up a wallet using SolFlare, how to send and
+receive SOL tokens, and how to create and manage a stake account.
+
+## Getting Started
+
+Go to https://www.solflare.com in a supported browser.  Most popular web browsers
+should work when interacting with a Keystore File, but currently only
+Chrome and Brave are supported when interacting with a Ledger Nano S.
+
+### Using a Keystore File
+
+#### Create a new Keystore File
+To create a wallet with a Keystore file, click on "Create a Wallet" and select
+"Using Keystore File".  Follow the prompts to create a password which will be
+used to encrypt your Keystore file, and then to download the new file to your
+computer.  You will be prompted to then upload the Keystore file back to the site
+to verify that the download was saved correctly.
+
+**NOTE: If you lose your Keystore file or the password used to encrypt it, any
+funds in that wallet will be lost permanently.  Neither the Solana team nor the
+SolFlare developers can help you recover lost keys.**
+
+You may want to consider saving a backup copy of your Keystore file on an
+external drive separate from your main computer, and storing your password in a
+separate location.
+
+#### Access your wallet with a Keystore File
+To use SolFlare with a previously created Keystore file, click on
+"Access a Wallet" and select "Using Keystore File".  If you just created a new
+Keystore file, you will be taken to the Access page directly.
+You will be prompted to enter the password and upload your Keystore file,
+then you will be taken to the wallet interface main page.
+
+### Using a Ledger Nano S hardware wallet
+
+#### Initial Device Setup
+To use a Ledger Nano S with SolFlare, first ensure you have
+[set up your Nano S](ledger-live.md) and have [installed the latest version of
+the Solana app](ledger-live.md#upgrade-to-the-latest-version-of-the-solana-app)
+on your device.
+
+#### Select a Ledger address to access
+Plug in your Nano S and open the Solana app.  Acknowledge a message of "Pending
+Ledger Review" by tapping both buttons at once so the device screen displays
+"Application is Ready".
+
+From the SolFlare home page, click "Access a Wallet" then select "Using Ledger
+Nano S".  Under "Select derivation path", select the only option:
+
+```Solana - 44`/501`/```
+
+Note: Your browser may prompt you to ask if SolFlare may communicate with your
+Ledger device.  Click to allow this.
+
+Select an address to interact with from the lower drop down box then click "Access".
+
+The Ledger device can derive a large number of private keys and associated
+public addresses. This allows you to manage and interact with an arbitrary
+number of different accounts from the same device.
+
+If you deposit funds to an address derived from your Ledger device,
+make sure to access the same address when using SolFlare to be able to access
+those funds.  If you connect to the incorrect address,
+simply click Logout and re-connect with the correct address.
+
+## Select a Network
+
+Solana maintains [three distinct networks](../clusters.md), each of which has
+its own purpose in supporting the Solana ecosystem.  Mainnet Beta is selected by
+default on SolFlare, as this is the permanent network where exchanges and other
+production apps are deployed.  To select a different network, click on the name
+of the currently selected network at the top of the wallet dashboard, either
+Mainnet, Testnet or Devnet, then click on the name of the network you wish to be
+using.
+
+## Sending and Receiving SOL Tokens
+
+### Receiving
+To receive tokens into your wallet, someone must transfer some to your wallet's
+address.  The address is displayed at the top-left on the screen, and you can
+click the Copy icon to copy the address and provide it to whoever is sending you
+tokens.  If you hold tokens in a different wallet or on an exchange, you can
+withdraw to this address as well.  Once the transfer is made, the balance shown
+on SolFlare should update within a few seconds.
+
+### Sending
+Once you have some tokens at your wallet address, you can send them to any other
+wallet address or an exchange deposit address by clicking "Transfer SOL" in the
+upper-right corner.  Enter the recipient address and the amount of SOL to
+transfer and click "Submit".  You will be prompted to confirm the details of the
+transaction before you [use your key to sign the transaction](#signing-a-transaction)
+and then it will be submitted to the network.
+
+## Staking SOL Tokens
+SolFlare supports creating and managing stake accounts and delegations.  To learn
+about how staking on Solana works in general, check out our
+[Staking Guide](../staking.md).
+
+### Create a Stake Account
+You can use some of the SOL tokens in your wallet to create a new stake account.
+From the wallet main page click "Staking" at the top of the page.  In the upper-
+right, click "Create Account".  Enter the amount of SOL you want to use to
+fund your new stake account.  This amount will be withdrawn from your wallet
+and transfered to the stake account.  Do not transfer your entire wallet balance
+to a stake account, as the wallet is still used to pay any transaction fees
+associated with your stake account.  Consider leaving at least 1 SOL in your
+wallet account.
+
+After you submit and [sign the transaction](#signing-a-transaction) you will see
+your new stake account appear in the box labeled "Your Staking Accounts".
+
+Stake accounts created on SolFlare set your wallet address as the
+[staking and withdrawing authority](../staking/stake-accounts.md#understanding-account-authorities)
+for your new account, which gives your wallet's key the authority to sign
+for any transactions related to the new stake account.
+
+### View your Stake Accounts
+On the main Wallet dashboard page or on the Staking dashboard page, your stake
+accounts will be visible in the "Your Staking Accounts" box.  Stake accounts
+exist at a different address from your wallet.
+
+SolFlare will locate any display all stake accounts on the
+[selected network](#select-a-network)
+for which your wallet address is assigned as the
+[stake authority](../staking/stake-accounts.md#understanding-account-authorities).
+Stake accounts that were created outside of SolFlare will also be displayed and
+can be managed as long as the wallet you logged in with is assigned as the stake
+authority.
+
+### Delegate tokens in a Stake Account
+Once you have [selected a validator](../staking.md#select-a-validator), you may
+delegate the tokens in one of your stake accounts to them.  From the Staking
+dashboard, click "Delegate" at the right side of a displayed stake account.
+Select the validator you wish to delegate to from the drop down list and click
+Delegate.
+
+To un-delegate your staked tokens (also called deactivating your stake), the
+process is similar.  On the Staking page, at the right side of a delegated stake
+account, click the "Undelegate" button and follow the prompts.
+
+### Split a Stake Account
+You may split an existing stake account into two stake accounts.  Click on the
+address of a stake account controlled by your wallet, and under the Actions bar,
+click "Split".  Specify the amount of SOL tokens you want to split.  This will be
+the amount of tokens in your new stake account and your existing stake account
+balance will be reduced by the same amount.  Splitting your stake account
+allows you to delegate to multiple different validators with different amounts
+of tokens. You may split a stake account as many times as you want, to create
+as many stake accounts as you want.
+
+## Signing a Transaction
+Any time you submit a transaction such as sending tokens to another wallet or
+delegating stake, you need to use your private key to sign the transaction so
+it will be accepted by the network.
+
+### Using a Keystore File
+If you accessed your wallet using a Keystore file, you will be prompted to enter
+your password any time the key is needed to sign a transaction.
+
+### Using a Ledger Nano S
+If you accessed your wallet with a Ledger Nano S, you will be prompted to confirm
+the pending transaction details on your device whenever the key is needed to sign.
+On the Nano S, use the left and right buttons to view and confirm all of the
+transaction details.  If everything looks correct, keep clicking the right button
+until the screen shows "Approve".  Click both buttons to approve the transaction.
+If something looks incorrect, press the right button once more so the screen shows
+"Reject" and press both buttons to reject the transaction.  After you approve
+or reject a transaction, you will see this reflected on the SolFlare page.

docs/src/wallet-guide/web-wallets.md

@@ -0,0 +1,10 @@
+---
+title: Web Wallets
+---
+
+Solana is supported by the following web wallets.
+
+## SolFlare
+[SolFlare.com](https://solflare.com/) is a community-created non-custodial
+web wallet that was built specifically for Solana.  Check out our guide for
+[using SolFlare](solflare.md).

dos/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-dos"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,14 +13,14 @@ clap = "2.33.1"
 log = "0.4.8"
 rand = "0.7.0"
 rayon = "1.3.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-core = { path = "../core", version = "1.3.4" }
-solana-ledger = { path = "../ledger", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-net-utils = { path = "../net-utils", version = "1.3.4" }
-solana-runtime = { path = "../runtime", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-core = { path = "../core", version = "1.3.6" }
+solana-ledger = { path = "../ledger", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-net-utils = { path = "../net-utils", version = "1.3.6" }
+solana-runtime = { path = "../runtime", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

dos/src/main.rs

@@ -149,6 +149,7 @@ fn main() {
 
     info!("Finding cluster entry: {:?}", entrypoint_addr);
     let (nodes, _validators) = discover(
+        None,
         Some(&entrypoint_addr),
         None,
         Some(60),

download-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-download-utils"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana Download Utils"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,8 +14,8 @@ console = "0.11.3"
 indicatif = "0.15.0"
 log = "0.4.8"
 reqwest = { version = "0.10.6", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-runtime = { path = "../runtime", version = "1.3.4" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-runtime = { path = "../runtime", version = "1.3.6" }
 tar = "0.4.28"
 
 [lib]

explorer/wasm/Cargo.lock

@@ -98,7 +98,7 @@ dependencies = [
 
 [[package]]
 name = "solana-sdk-wasm"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "bincode",
  "bs58",

explorer/wasm/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-sdk-wasm"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana SDK Wasm"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

faucet/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-faucet"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana Faucet"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -16,11 +16,11 @@ clap = "2.33"
 log = "0.4.8"
 serde = "1.0.112"
 serde_derive = "1.0.103"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-metrics = { path = "../metrics", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-metrics = { path = "../metrics", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
 tokio = "0.1"
 tokio-codec = "0.1"
 

fetch-perf-libs.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-PERF_LIBS_VERSION=v0.19.1
+PERF_LIBS_VERSION=v0.19.3
 VERSION=$PERF_LIBS_VERSION-1
 
 set -e

fetch-spl.sh

@@ -10,10 +10,11 @@ fetch_program() {
   declare name=$1
   declare version=$2
   declare address=$3
+  declare loader=$4
 
   declare so=spl_$name-$version.so
 
-  genesis_args+=(--bpf-program "$address" "$so")
+  genesis_args+=(--bpf-program "$address" "$loader" "$so")
 
   if [[ -r $so ]]; then
     return
@@ -36,8 +37,8 @@ fetch_program() {
 
 }
 
-fetch_program token 1.0.0 TokenSVp5gheXUvJ6jGWGeCsgPKgnE3YgdGKRVCMY9o
-fetch_program memo  1.0.0 Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo
+fetch_program token 2.0.3 TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA BPFLoader1111111111111111111111111111111111
+fetch_program memo  1.0.0 Memo1UhkJRfHyvLMcVucJwxXeuD728EqVDDwQDxFMNo BPFLoader1111111111111111111111111111111111
 
 echo "${genesis_args[@]}" > spl-genesis-args.sh
 

genesis-programs/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-genesis-programs"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana genesis programs"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,12 +10,12 @@ edition = "2018"
 
 [dependencies]
 log = { version = "0.4.8" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.3.4" }
-solana-budget-program = { path = "../programs/budget", version = "1.3.4" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.3.4" }
-solana-runtime = { path = "../runtime", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-vest-program = { path = "../programs/vest", version = "1.3.4" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.3.6" }
+solana-budget-program = { path = "../programs/budget", version = "1.3.6" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.3.6" }
+solana-runtime = { path = "../runtime", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-vest-program = { path = "../programs/vest", version = "1.3.6" }
 
 [lib]
 crate-type = ["lib"]

genesis-programs/src/lib.rs

@@ -1,7 +1,3 @@
-use solana_sdk::{
-    clock::Epoch, genesis_config::OperatingMode, inflation::Inflation, pubkey::Pubkey,
-};
-
 #[macro_use]
 extern crate solana_bpf_loader_program;
 #[macro_use]
@@ -13,6 +9,13 @@ extern crate solana_vest_program;
 
 use log::*;
 use solana_runtime::bank::{Bank, EnteredEpochCallback};
+use solana_sdk::{
+    clock::{Epoch, GENESIS_EPOCH},
+    entrypoint_native::{ErasedProcessInstructionWithContext, ProcessInstructionWithContext},
+    genesis_config::OperatingMode,
+    inflation::Inflation,
+    pubkey::Pubkey,
+};
 
 pub fn get_inflation(operating_mode: OperatingMode, epoch: Epoch) -> Option<Inflation> {
     match operating_mode {
@@ -36,80 +39,145 @@ pub fn get_inflation(operating_mode: OperatingMode, epoch: Epoch) -> Option<Infl
         OperatingMode::Stable => match epoch {
             // No inflation at epoch 0
             0 => Some(Inflation::new_disabled()),
-            // Inflation starts
-            // The epoch of Epoch::MAX is a placeholder and is expected to be reduced in
-            // a future hard fork.
+            // Inflation starts The epoch of Epoch::MAX is a placeholder and is
+            // expected to be reduced in a future hard fork.
             Epoch::MAX => Some(Inflation::default()),
             _ => None,
         },
     }
 }
 
-pub fn get_programs(operating_mode: OperatingMode, epoch: Epoch) -> Option<Vec<(String, Pubkey)>> {
-    match operating_mode {
-        OperatingMode::Development => {
-            if epoch == 0 {
-                Some(vec![
-                    // Enable all Stable programs
-                    solana_bpf_loader_program!(),
-                    solana_vest_program!(),
-                    // Programs that are only available in Development mode
-                    solana_budget_program!(),
-                    solana_exchange_program!(),
-                ])
-            } else {
-                None
-            }
+enum Program {
+    Native((String, Pubkey)),
+    BuiltinLoader((String, Pubkey, ProcessInstructionWithContext)),
+}
+
+impl std::fmt::Debug for Program {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        #[derive(Debug)]
+        enum Program {
+            Native((String, Pubkey)),
+            BuiltinLoader((String, Pubkey, String)),
         }
-        OperatingMode::Stable => {
-            if epoch == std::u64::MAX - 1 {
-                // The epoch of std::u64::MAX - 1 is a placeholder and is expected to be reduced in
-                // a future hard fork.
-                Some(vec![solana_bpf_loader_program!()])
-            } else if epoch == std::u64::MAX {
-                // The epoch of std::u64::MAX is a placeholder and is expected to be reduced in a
-                // future hard fork.
-                Some(vec![solana_vest_program!()])
-            } else {
-                None
+        let program = match self {
+            crate::Program::Native((string, pubkey)) => Program::Native((string.clone(), *pubkey)),
+            crate::Program::BuiltinLoader((string, pubkey, instruction)) => {
+                let erased: ErasedProcessInstructionWithContext = *instruction;
+                Program::BuiltinLoader((string.clone(), *pubkey, format!("{:p}", erased)))
             }
-        }
-        OperatingMode::Preview => {
-            if epoch == 0 {
-                Some(vec![solana_bpf_loader_program!()])
-            } else if epoch == std::u64::MAX {
-                // The epoch of std::u64::MAX is a placeholder and is expected to be reduced in a
-                // future hard fork.
-                Some(vec![solana_vest_program!()])
-            } else {
-                None
-            }
-        }
+        };
+        write!(f, "{:?}", program)
     }
 }
 
+// given operating_mode and epoch, return the entire set of enabled programs
+fn get_programs(operating_mode: OperatingMode) -> Vec<(Program, Epoch)> {
+    let mut programs = vec![];
+
+    match operating_mode {
+        OperatingMode::Development => {
+            // Programs used for testing
+            programs.extend(
+                vec![
+                    Program::BuiltinLoader(solana_bpf_loader_program!()),
+                    Program::BuiltinLoader(solana_bpf_loader_deprecated_program!()),
+                    Program::Native(solana_vest_program!()),
+                    Program::Native(solana_budget_program!()),
+                    Program::Native(solana_exchange_program!()),
+                ]
+                .into_iter()
+                .map(|program| (program, 0))
+                .collect::<Vec<_>>(),
+            );
+        }
+        OperatingMode::Preview => {
+            programs.extend(
+                vec![
+                    Program::BuiltinLoader(solana_bpf_loader_program!()),
+                    Program::BuiltinLoader(solana_bpf_loader_deprecated_program!()),
+                ]
+                .into_iter()
+                .map(|program| (program, 0))
+                .collect::<Vec<_>>(),
+            );
+            // The epoch of Epoch::max_value() is a placeholder and is expected
+            // to be reduced in a future network update.
+            programs.extend(
+                vec![Program::Native(solana_vest_program!())]
+                    .into_iter()
+                    .map(|program| (program, Epoch::MAX))
+                    .collect::<Vec<_>>(),
+            );
+        }
+        OperatingMode::Stable => {
+            programs.extend(vec![(
+                Program::BuiltinLoader(solana_bpf_loader_deprecated_program!()),
+                34,
+            )]);
+            // The epoch of std::u64::MAX is a placeholder and is expected
+            // to be reduced in a future network update.
+            programs.extend(
+                vec![
+                    Program::BuiltinLoader(solana_bpf_loader_program!()),
+                    Program::Native(solana_vest_program!()),
+                ]
+                .into_iter()
+                .map(|program| (program, Epoch::MAX))
+                .collect::<Vec<_>>(),
+            );
+        }
+    };
+
+    programs
+}
+
+pub fn get_native_programs_for_genesis(operating_mode: OperatingMode) -> Vec<(String, Pubkey)> {
+    let mut native_programs = vec![];
+    for (program, start_epoch) in get_programs(operating_mode) {
+        if let Program::Native((string, key)) = program {
+            if start_epoch == GENESIS_EPOCH {
+                native_programs.push((string, key));
+            }
+        }
+    }
+    native_programs
+}
+
 pub fn get_entered_epoch_callback(operating_mode: OperatingMode) -> EnteredEpochCallback {
-    Box::new(move |bank: &mut Bank| {
-        info!(
-            "Entering epoch {} with operating_mode {:?}",
-            bank.epoch(),
-            operating_mode
-        );
+    Box::new(move |bank: &mut Bank, initial: bool| {
+        // Be careful to add arbitrary logic here; this should be idempotent and can be called
+        // at arbitrary point in an epoch not only epoch boundaries.
+        // This is because this closure need to be executed immediately after snapshot restoration,
+        // in addition to usual epoch boundaries
+        // In other words, this callback initializes some skip(serde) fields, regardless
+        // frozen or not
+
         if let Some(inflation) = get_inflation(operating_mode, bank.epoch()) {
             info!("Entering new epoch with inflation {:?}", inflation);
             bank.set_inflation(inflation);
         }
-        if let Some(new_programs) = get_programs(operating_mode, bank.epoch()) {
-            for (name, program_id) in new_programs.iter() {
-                info!("Registering {} at {}", name, program_id);
-                bank.add_native_program(name, program_id);
+        for (program, start_epoch) in get_programs(operating_mode) {
+            let should_populate =
+                initial && bank.epoch() >= start_epoch || !initial && bank.epoch() == start_epoch;
+            if should_populate {
+                match program {
+                    Program::Native((name, program_id)) => {
+                        bank.add_native_program(&name, &program_id);
+                    }
+                    Program::BuiltinLoader((
+                        name,
+                        program_id,
+                        process_instruction_with_context,
+                    )) => {
+                        bank.add_builtin_loader(
+                            &name,
+                            program_id,
+                            process_instruction_with_context,
+                        );
+                    }
+                }
             }
         }
-        if OperatingMode::Stable == operating_mode {
-            bank.set_cross_program_support(bank.epoch() >= 63);
-        } else {
-            bank.set_cross_program_support(true);
-        }
     })
 }
 
@@ -121,8 +189,13 @@ mod tests {
     #[test]
     fn test_id_uniqueness() {
         let mut unique = HashSet::new();
-        let ids = get_programs(OperatingMode::Development, 0).unwrap();
-        assert!(ids.into_iter().all(move |id| unique.insert(id)));
+        let programs = get_programs(OperatingMode::Development);
+        for (program, _start_epoch) in programs {
+            match program {
+                Program::Native((name, id)) => assert!(unique.insert((name, id))),
+                Program::BuiltinLoader((name, id, _)) => assert!(unique.insert((name, id))),
+            }
+        }
     }
 
     #[test]
@@ -136,11 +209,15 @@ mod tests {
 
     #[test]
     fn test_development_programs() {
+        assert_eq!(get_programs(OperatingMode::Development).len(), 5);
+    }
+
+    #[test]
+    fn test_native_development_programs() {
         assert_eq!(
-            get_programs(OperatingMode::Development, 0).unwrap().len(),
-            4
+            get_native_programs_for_genesis(OperatingMode::Development).len(),
+            3
         );
-        assert_eq!(get_programs(OperatingMode::Development, 1), None);
     }
 
     #[test]
@@ -158,8 +235,6 @@ mod tests {
 
     #[test]
     fn test_softlaunch_programs() {
-        assert_eq!(get_programs(OperatingMode::Stable, 1), None);
-        assert!(get_programs(OperatingMode::Stable, std::u64::MAX - 1).is_some());
-        assert!(get_programs(OperatingMode::Stable, std::u64::MAX).is_some());
+        assert!(!get_programs(OperatingMode::Stable).is_empty());
     }
 }

genesis/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-genesis"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,15 +15,15 @@ chrono = "0.4"
 serde = "1.0.112"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-genesis-programs = { path = "../genesis-programs", version = "1.3.4" }
-solana-ledger = { path = "../ledger", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-runtime = { path = "../runtime", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-stake-program = { path = "../programs/stake", version = "1.3.4" }
-solana-vote-program = { path = "../programs/vote", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.3.6" }
+solana-ledger = { path = "../ledger", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-runtime = { path = "../runtime", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-stake-program = { path = "../programs/stake", version = "1.3.6" }
+solana-vote-program = { path = "../programs/vote", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
 tempfile = "3.1.0"
 
 [[bin]]

genesis/src/main.rs

@@ -12,7 +12,7 @@ use solana_ledger::{
 use solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE;
 use solana_sdk::{
     account::Account,
-    bpf_loader, clock,
+    clock,
     epoch_schedule::EpochSchedule,
     fee_calculator::FeeRateGovernor,
     genesis_config::{GenesisConfig, OperatingMode},
@@ -353,7 +353,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                 .long("bpf-program")
                 .value_name("ADDRESS BPF_PROGRAM.SO")
                 .takes_value(true)
-                .number_of_values(2)
+                .number_of_values(3)
                 .multiple(true)
                 .help("Install a BPF program at the given address"),
         )
@@ -468,7 +468,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
     );
 
     let native_instruction_processors =
-        solana_genesis_programs::get_programs(operating_mode, 0).unwrap_or_else(Vec::new);
+        solana_genesis_programs::get_native_programs_for_genesis(operating_mode);
     let inflation = solana_genesis_programs::get_inflation(operating_mode, 0).unwrap();
 
     let mut genesis_config = GenesisConfig {
@@ -553,14 +553,19 @@ fn main() -> Result<(), Box<dyn error::Error>> {
 
     if let Some(values) = matches.values_of("bpf_program") {
         let values: Vec<&str> = values.collect::<Vec<_>>();
-        for address_program in values.chunks(2) {
-            match address_program {
-                [address, program] => {
+        for address_loader_program in values.chunks(3) {
+            match address_loader_program {
+                [address, loader, program] => {
                     let address = address.parse::<Pubkey>().unwrap_or_else(|err| {
                         eprintln!("Error: invalid address {}: {}", address, err);
                         process::exit(1);
                     });
 
+                    let loader = loader.parse::<Pubkey>().unwrap_or_else(|err| {
+                        eprintln!("Error: invalid loader {}: {}", loader, err);
+                        process::exit(1);
+                    });
+
                     let mut program_data = vec![];
                     File::open(program)
                         .and_then(|mut file| file.read_to_end(&mut program_data))
@@ -574,7 +579,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                             lamports: genesis_config.rent.minimum_balance(program_data.len()),
                             data: program_data,
                             executable: true,
-                            owner: bpf_loader::id(),
+                            owner: loader,
                             rent_epoch: 0,
                         },
                     );

gossip/Cargo.toml

@@ -3,20 +3,20 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-gossip"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-core = { path = "../core", version = "1.3.4" }
-solana-client = { path = "../client", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-net-utils = { path = "../net-utils", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-core = { path = "../core", version = "1.3.6" }
+solana-client = { path = "../client", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-net-utils = { path = "../net-utils", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

gossip/src/main.rs

@@ -4,13 +4,19 @@ use clap::{
     crate_description, crate_name, value_t, value_t_or_exit, App, AppSettings, Arg, ArgMatches,
     SubCommand,
 };
-use solana_clap_utils::input_validators::{is_port, is_pubkey};
+use solana_clap_utils::{
+    input_parsers::keypair_of,
+    input_validators::{is_keypair_or_ask_keyword, is_port, is_pubkey},
+};
 use solana_client::rpc_client::RpcClient;
 use solana_core::{contact_info::ContactInfo, gossip_service::discover};
 use solana_sdk::pubkey::Pubkey;
-use std::error;
-use std::net::{IpAddr, Ipv4Addr, SocketAddr};
-use std::process::exit;
+use std::{
+    error,
+    net::{IpAddr, Ipv4Addr, SocketAddr},
+    process::exit,
+    sync::Arc,
+};
 
 fn parse_matches() -> ArgMatches<'static> {
     let shred_version_arg = Arg::with_name("shred_version")
@@ -88,7 +94,17 @@ fn parse_matches() -> ArgMatches<'static> {
                         .value_name("HOST")
                         .takes_value(true)
                         .validator(solana_net_utils::is_host)
-                        .help("Gossip DNS name or IP address for the node [default: ask --entrypoint, or 127.0.0.1 when --entrypoint is not provided]"),
+                        .help("Gossip DNS name or IP address for the node \
+                               [default: ask --entrypoint, or 127.0.0.1 when --entrypoint is not provided]"),
+                )
+                .arg(
+                    Arg::with_name("identity")
+                        .short("i")
+                        .long("identity")
+                        .value_name("PATH")
+                        .takes_value(true)
+                        .validator(is_keypair_or_ask_keyword)
+                        .help("Identity keypair [default: ephemeral keypair]"),
                 )
                 .arg(
                     Arg::with_name("num_nodes")
@@ -230,6 +246,7 @@ fn process_spy(matches: &ArgMatches) -> std::io::Result<()> {
         .value_of("node_pubkey")
         .map(|pubkey_str| pubkey_str.parse::<Pubkey>().unwrap());
     let shred_version = value_t_or_exit!(matches, "shred_version", u16);
+    let identity_keypair = keypair_of(&matches, "identity").map(Arc::new);
 
     let entrypoint_addr = parse_entrypoint(matches);
 
@@ -247,6 +264,7 @@ fn process_spy(matches: &ArgMatches) -> std::io::Result<()> {
     );
 
     let (_all_peers, validators) = discover(
+        identity_keypair,
         entrypoint_addr.as_ref(),
         num_nodes,
         timeout,
@@ -277,6 +295,7 @@ fn process_rpc_url(matches: &ArgMatches) -> std::io::Result<()> {
     let timeout = value_t_or_exit!(matches, "timeout", u64);
     let shred_version = value_t_or_exit!(matches, "shred_version", u16);
     let (_all_peers, validators) = discover(
+        None,
         entrypoint_addr.as_ref(),
         Some(1),
         Some(timeout),
@@ -321,6 +340,7 @@ fn process_stop(matches: &ArgMatches) -> Result<(), Box<dyn error::Error>> {
         .parse::<Pubkey>()
         .unwrap();
     let (_all_peers, validators) = discover(
+        None,
         entrypoint_addr.as_ref(),
         None,
         None,

install/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-install"
 description = "The solana cluster software installer"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -24,12 +24,12 @@ reqwest = { version = "0.10.6", default-features = false, features = ["blocking"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-client = { path = "../client", version = "1.3.4" }
-solana-config-program = { path = "../programs/config", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-client = { path = "../client", version = "1.3.6" }
+solana-config-program = { path = "../programs/config", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
 semver = "0.9.0"
 tar = "0.4.28"
 tempdir = "0.3.7"

keygen/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-keygen"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana key generation utility"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,11 +13,11 @@ bs58 = "0.3.1"
 clap = "2.33"
 dirs = "2.0.2"
 num_cpus = "1.13.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-cli-config = { path = "../cli-config", version = "1.3.4" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-cli-config = { path = "../cli-config", version = "1.3.6" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
 tiny-bip39 = "0.7.0"
 
 [[bin]]

ledger-tool/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-ledger-tool"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -19,18 +19,18 @@ log = { version = "0.4.8" }
 regex = "1"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-cli = { path = "../cli", version = "1.3.4" }
-solana-ledger = { path = "../ledger", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-measure = { path = "../measure", version = "1.3.4" }
-solana-runtime = { path = "../runtime", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-stake-program = { path = "../programs/stake", version = "1.3.4" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "1.3.4" }
-solana-transaction-status = { path = "../transaction-status", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
-solana-vote-program = { path = "../programs/vote", version = "1.3.4" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-cli = { path = "../cli", version = "1.3.6" }
+solana-ledger = { path = "../ledger", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-measure = { path = "../measure", version = "1.3.6" }
+solana-runtime = { path = "../runtime", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-stake-program = { path = "../programs/stake", version = "1.3.6" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "1.3.6" }
+solana-transaction-status = { path = "../transaction-status", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
+solana-vote-program = { path = "../programs/vote", version = "1.3.6" }
 tempfile = "3.1.0"
 tokio = { version = "0.2.22", features = ["full"] }
 

ledger-tool/src/main.rs

@@ -1028,6 +1028,12 @@ fn main() {
                     .takes_value(false)
                     .help("Include sysvars too"),
             )
+            .arg(
+                Arg::with_name("exclude_account_data")
+                    .long("exclude-account-data")
+                    .takes_value(false)
+                    .help("Exclude account data (useful for large number of accounts)"),
+            )
             .arg(&max_genesis_archive_unpacked_size_arg)
         ).subcommand(
             SubCommand::with_name("capitalization")
@@ -1537,6 +1543,7 @@ fn main() {
             };
             let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
             let include_sysvars = arg_matches.is_present("include_sysvars");
+            let exclude_account_data = arg_matches.is_present("exclude_account_data");
             match load_bank_forks(
                 arg_matches,
                 &ledger_path,
@@ -1554,21 +1561,26 @@ fn main() {
                     });
 
                     let accounts: BTreeMap<_, _> = bank
-                        .get_program_accounts(None)
+                        .get_all_accounts_with_modified_slots()
                         .into_iter()
-                        .filter(|(pubkey, _account)| {
+                        .filter(|(pubkey, _account, _slot)| {
                             include_sysvars || !solana_sdk::sysvar::is_sysvar_id(pubkey)
                         })
+                        .map(|(pubkey, account, slot)| (pubkey, (account, slot)))
                         .collect();
 
                     println!("---");
-                    for (pubkey, account) in accounts.into_iter() {
+                    for (pubkey, (account, slot)) in accounts.into_iter() {
                         let data_len = account.data.len();
                         println!("{}:", pubkey);
                         println!("  - balance: {} SOL", lamports_to_sol(account.lamports));
                         println!("  - owner: '{}'", account.owner);
                         println!("  - executable: {}", account.executable);
-                        println!("  - data: '{}'", bs58::encode(account.data).into_string());
+                        println!("  - slot: {}", slot);
+                        println!("  - rent_epoch: {}", account.rent_epoch);
+                        if !exclude_account_data {
+                            println!("  - data: '{}'", bs58::encode(account.data).into_string());
+                        }
                         println!("  - data_len: {}", data_len);
                     }
                 }

ledger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-ledger"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana ledger"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -29,18 +29,18 @@ reed-solomon-erasure = { version = "4.0.2", features = ["simd-accel"] }
 serde = "1.0.112"
 serde_bytes = "0.11.4"
 sha2 = "0.8.2"
-solana-transaction-status = { path = "../transaction-status", version = "1.3.4" }
-solana-genesis-programs = { path = "../genesis-programs", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-measure = { path = "../measure", version = "1.3.4" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.3.4" }
-solana-metrics = { path = "../metrics", version = "1.3.4" }
-solana-perf = { path = "../perf", version = "1.3.4" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.3.4" }
-solana-runtime = { path = "../runtime", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-stake-program = { path = "../programs/stake", version = "1.3.4" }
-solana-vote-program = { path = "../programs/vote", version = "1.3.4" }
+solana-transaction-status = { path = "../transaction-status", version = "1.3.6" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-measure = { path = "../measure", version = "1.3.6" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.3.6" }
+solana-metrics = { path = "../metrics", version = "1.3.6" }
+solana-perf = { path = "../perf", version = "1.3.6" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.3.6" }
+solana-runtime = { path = "../runtime", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-stake-program = { path = "../programs/stake", version = "1.3.6" }
+solana-vote-program = { path = "../programs/vote", version = "1.3.6" }
 tempfile = "3.1.0"
 thiserror = "1.0"
 trees = "0.2.1"
@@ -55,7 +55,7 @@ features = ["lz4"]
 [dev-dependencies]
 assert_matches = "1.3.0"
 matches = "0.1.6"
-solana-budget-program = { path = "../programs/budget", version = "1.3.4" }
+solana-budget-program = { path = "../programs/budget", version = "1.3.6" }
 
 [lib]
 crate-type = ["lib"]

ledger/src/blockstore.rs

@@ -350,18 +350,39 @@ impl Blockstore {
         Ok((blockstore, signal_receiver, completed_slots_receiver))
     }
 
-    pub fn add_tree(&self, forks: Tree<Slot>, is_orphan: bool, is_slot_complete: bool) {
+    pub fn add_tree(
+        &self,
+        forks: Tree<Slot>,
+        is_orphan: bool,
+        is_slot_complete: bool,
+        num_ticks: u64,
+        starting_hash: Hash,
+    ) {
         let mut walk = TreeWalk::from(forks);
+        let mut blockhashes = HashMap::new();
         while let Some(visit) = walk.get() {
             let slot = visit.node().data;
             if self.meta(slot).unwrap().is_some() && self.orphan(slot).unwrap().is_none() {
-                // If slot exists and is not an orphan, then skip it
+                // If slot exists in blockstore and is not an orphan, then skip it
                 walk.forward();
                 continue;
             }
             let parent = walk.get_parent().map(|n| n.data);
             if parent.is_some() || !is_orphan {
-                let entries = create_ticks(2, 0, Hash::default());
+                let parent_hash = parent
+                    // parent won't exist for first node in a tree where
+                    // `is_orphan == true`
+                    .and_then(|parent| blockhashes.get(&parent))
+                    .unwrap_or(&starting_hash);
+                let mut entries = create_ticks(
+                    num_ticks * (std::cmp::max(1, slot - parent.unwrap_or(slot))),
+                    0,
+                    *parent_hash,
+                );
+                blockhashes.insert(slot, entries.last().unwrap().hash);
+                if !is_slot_complete {
+                    entries.pop().unwrap();
+                }
                 let shreds = entries_to_test_shreds(
                     entries.clone(),
                     slot,
@@ -407,6 +428,16 @@ impl Blockstore {
         self.orphans_cf.get(slot)
     }
 
+    // Get max root or 0 if it doesn't exist
+    pub fn max_root(&self) -> Slot {
+        self.db
+            .iter::<cf::Root>(IteratorMode::End)
+            .expect("Couldn't get rooted iterator for max_root()")
+            .next()
+            .map(|(slot, _)| slot)
+            .unwrap_or(0)
+    }
+
     pub fn slot_meta_iterator<'a>(
         &'a self,
         slot: Slot,
@@ -3257,7 +3288,7 @@ fn adjust_ulimit_nofile() -> Result<()> {
 fn adjust_ulimit_nofile() -> Result<()> {
     // Rocks DB likes to have many open files.  The default open file descriptor limit is
     // usually not enough
-    let desired_nofile = 65000;
+    let desired_nofile = 500000;
 
     fn get_nofile() -> libc::rlimit {
         let mut nofile = libc::rlimit {
@@ -3280,7 +3311,10 @@ fn adjust_ulimit_nofile() -> Result<()> {
             );
 
             if cfg!(target_os = "macos") {
-                error!("On mac OS you may need to run |sudo launchctl limit maxfiles 65536 200000| first");
+                error!(
+                    "On mac OS you may need to run |sudo launchctl limit maxfiles {} {}| first",
+                    desired_nofile, desired_nofile,
+                );
             }
             return Err(BlockstoreError::UnableToSetOpenFileDescriptorLimit);
         }

ledger/src/blockstore_processor.rs

@@ -18,6 +18,7 @@ use solana_runtime::{
     bank::{Bank, TransactionBalancesSet, TransactionProcessResult, TransactionResults},
     bank_forks::BankForks,
     bank_utils,
+    commitment::VOTE_THRESHOLD_SIZE,
     transaction_batch::TransactionBatch,
     transaction_utils::OrderedIterator,
     vote_sender_types::ReplayVoteSender,
@@ -31,9 +32,10 @@ use solana_sdk::{
     timing::duration_as_ms,
     transaction::{Result, Transaction, TransactionError},
 };
+use solana_vote_program::vote_state::VoteState;
 use std::{
     cell::RefCell,
-    collections::HashMap,
+    collections::{BTreeMap, HashMap},
     path::PathBuf,
     result,
     sync::Arc,
@@ -307,6 +309,14 @@ pub struct ProcessOptions {
     pub frozen_accounts: Vec<Pubkey>,
 }
 
+fn initiate_callback(mut bank: &mut Arc<Bank>, genesis_config: &GenesisConfig) {
+    Arc::get_mut(&mut bank)
+        .unwrap()
+        .initiate_entered_epoch_callback(solana_genesis_programs::get_entered_epoch_callback(
+            genesis_config.operating_mode,
+        ));
+}
+
 pub fn process_blockstore(
     genesis_config: &GenesisConfig,
     blockstore: &Blockstore,
@@ -323,15 +333,24 @@ pub fn process_blockstore(
     }
 
     // Setup bank for slot 0
-    let bank0 = Arc::new(Bank::new_with_paths(
+    let mut bank0 = Arc::new(Bank::new_with_paths(
         &genesis_config,
         account_paths,
         &opts.frozen_accounts,
     ));
+    initiate_callback(&mut bank0, genesis_config);
     info!("processing ledger for slot 0...");
     let recyclers = VerifyRecyclers::default();
     process_bank_0(&bank0, blockstore, &opts, &recyclers)?;
-    process_blockstore_from_root(genesis_config, blockstore, bank0, &opts, &recyclers, None)
+    do_process_blockstore_from_root(
+        genesis_config,
+        blockstore,
+        bank0,
+        &opts,
+        &recyclers,
+        None,
+        false,
+    )
 }
 
 // Process blockstore from a known root bank
@@ -342,6 +361,26 @@ pub fn process_blockstore_from_root(
     opts: &ProcessOptions,
     recyclers: &VerifyRecyclers,
     transaction_status_sender: Option<TransactionStatusSender>,
+) -> BlockstoreProcessorResult {
+    do_process_blockstore_from_root(
+        genesis_config,
+        blockstore,
+        bank,
+        opts,
+        recyclers,
+        transaction_status_sender,
+        true,
+    )
+}
+
+fn do_process_blockstore_from_root(
+    genesis_config: &GenesisConfig,
+    blockstore: &Blockstore,
+    mut bank: Arc<Bank>,
+    opts: &ProcessOptions,
+    recyclers: &VerifyRecyclers,
+    transaction_status_sender: Option<TransactionStatusSender>,
+    enable_callback: bool,
 ) -> BlockstoreProcessorResult {
     info!("processing ledger from slot {}...", bank.slot());
     let allocated = thread_mem_usage::Allocatedp::default();
@@ -353,9 +392,9 @@ pub fn process_blockstore_from_root(
     let now = Instant::now();
     let mut root = start_slot;
 
-    bank.set_entered_epoch_callback(solana_genesis_programs::get_entered_epoch_callback(
-        genesis_config.operating_mode,
-    ));
+    if enable_callback {
+        initiate_callback(&mut bank, genesis_config);
+    }
 
     if let Some(ref new_hard_forks) = opts.new_hard_forks {
         let hard_forks = bank.hard_forks();
@@ -454,6 +493,7 @@ pub fn verify_ticks(
 ) -> std::result::Result<(), BlockError> {
     let next_bank_tick_height = bank.tick_height() + entries.tick_count();
     let max_bank_tick_height = bank.max_tick_height();
+
     if next_bank_tick_height > max_bank_tick_height {
         warn!("Too many entry ticks found in slot: {}", bank.slot());
         return Err(BlockError::InvalidTickCount);
@@ -757,11 +797,18 @@ fn load_frozen_forks(
     transaction_status_sender: Option<TransactionStatusSender>,
 ) -> result::Result<Vec<Arc<Bank>>, BlockstoreProcessorError> {
     let mut initial_forks = HashMap::new();
+    let mut all_banks = HashMap::new();
     let mut last_status_report = Instant::now();
     let mut pending_slots = vec![];
     let mut last_root_slot = root_bank.slot();
     let mut slots_elapsed = 0;
     let mut txs = 0;
+    let blockstore_max_root = blockstore.max_root();
+    let max_root = std::cmp::max(root_bank.slot(), blockstore_max_root);
+    info!(
+        "load_frozen_forks() latest root from blockstore: {}, max_root: {}",
+        blockstore_max_root, max_root,
+    );
     process_next_slots(
         root_bank,
         root_meta,
@@ -810,14 +857,49 @@ fn load_frozen_forks(
         }
         txs += progress.num_txs;
 
-        if blockstore.is_root(slot) {
-            *root = slot;
-            leader_schedule_cache.set_root(&bank);
-            bank.squash();
-            pending_slots.clear();
-            initial_forks.clear();
-            last_root_slot = slot;
+        // Block must be frozen by this point, otherwise `process_single_slot` would
+        // have errored above
+        assert!(bank.is_frozen());
+        all_banks.insert(bank.slot(), bank.clone());
+
+        // If we've reached the last known root in blockstore, start looking
+        // for newer cluster confirmed roots
+        let new_root_bank = {
+            if *root == max_root {
+                supermajority_root_from_bank(&bank).and_then(|supermajority_root| {
+                    if supermajority_root > *root {
+                        // If there's a cluster confirmed root greater than our last
+                        // replayed root, then beccause the cluster confirmed root should
+                        // be descended from our last root, it must exist in `all_banks`
+                        let cluster_root_bank = all_banks.get(&supermajority_root).unwrap();
+
+                        // cluster root must be a descendant of our root, otherwise something
+                        // is drastically wrong
+                        assert!(cluster_root_bank.ancestors.contains_key(root));
+                        info!("blockstore processor found new cluster confirmed root: {}, observed in bank: {}", cluster_root_bank.slot(), bank.slot());
+                        Some(cluster_root_bank)
+                    } else {
+                        None
+                    }
+                })
+            } else if blockstore.is_root(slot) {
+                Some(&bank)
+            } else {
+                None
+            }
+        };
+
+        if let Some(new_root_bank) = new_root_bank {
+            *root = new_root_bank.slot();
+            last_root_slot = new_root_bank.slot();
+            leader_schedule_cache.set_root(&new_root_bank);
+            new_root_bank.squash();
+
+            // Filter out all non descendants of the new root
+            pending_slots.retain(|(_, pending_bank, _)| pending_bank.ancestors.contains_key(root));
+            initial_forks.retain(|_, fork_tip_bank| fork_tip_bank.ancestors.contains_key(root));
         }
+
         slots_elapsed += 1;
 
         trace!(
@@ -844,6 +926,51 @@ fn load_frozen_forks(
     Ok(initial_forks.values().cloned().collect::<Vec<_>>())
 }
 
+fn supermajority_root(roots: &BTreeMap<Slot, u64>, total_epoch_stake: u64) -> Option<Slot> {
+    // Find latest root
+    let mut total = 0;
+    for (root, stake) in roots.iter().rev() {
+        total += stake;
+        if total as f64 / total_epoch_stake as f64 > VOTE_THRESHOLD_SIZE {
+            return Some(*root);
+        }
+    }
+
+    None
+}
+
+fn supermajority_root_from_bank(bank: &Bank) -> Option<Slot> {
+    let roots: BTreeMap<Slot, u64> = bank
+        .vote_accounts()
+        .into_iter()
+        .filter_map(|(key, (stake, account))| {
+            if stake == 0 {
+                return None;
+            }
+
+            let vote_state = VoteState::from(&account);
+            if vote_state.is_none() {
+                warn!(
+                    "Unable to get vote_state from account {} in bank: {}",
+                    key,
+                    bank.slot()
+                );
+                return None;
+            }
+
+            vote_state
+                .unwrap()
+                .root_slot
+                .map(|root_slot| (root_slot, stake))
+        })
+        .collect();
+
+    let total_epoch_stake = bank.total_epoch_stake();
+
+    // Find latest root
+    supermajority_root(&roots, total_epoch_stake)
+}
+
 // Processes and replays the contents of a single slot, returns Error
 // if failed to play the slot
 fn process_single_slot(
@@ -871,6 +998,7 @@ fn process_single_slot(
     })?;
 
     bank.freeze(); // all banks handled by this routine are created from complete slots
+
     Ok(())
 }
 
@@ -951,7 +1079,7 @@ pub mod tests {
     use matches::assert_matches;
     use rand::{thread_rng, Rng};
     use solana_runtime::genesis_utils::{
-        create_genesis_config_with_vote_accounts, ValidatorVoteKeypairs,
+        self, create_genesis_config_with_vote_accounts, ValidatorVoteKeypairs,
     };
     use solana_sdk::account::Account;
     use solana_sdk::{
@@ -963,8 +1091,9 @@ pub mod tests {
         system_transaction,
         transaction::{Transaction, TransactionError},
     };
-    use solana_vote_program::vote_transaction;
+    use solana_vote_program::{vote_state::MAX_LOCKOUT_HISTORY, vote_transaction};
     use std::{collections::BTreeSet, sync::RwLock};
+    use trees::tr;
 
     #[test]
     fn test_process_blockstore_with_missing_hashes() {
@@ -1535,11 +1664,11 @@ pub mod tests {
         let blockstore =
             Blockstore::open(&ledger_path).expect("Expected to successfully open database ledger");
 
-        // Let last_slot be the number of slots in the first two epochs
+        // Let `last_slot` be the number of slots in the first two epochs
         let epoch_schedule = get_epoch_schedule(&genesis_config, Vec::new());
         let last_slot = epoch_schedule.get_last_slot_in_epoch(1);
 
-        // Create a single chain of slots with all indexes in the range [0, last_slot + 1]
+        // Create a single chain of slots with all indexes in the range [0, v + 1]
         for i in 1..=last_slot + 1 {
             last_entry_hash = fill_blockstore_slot_with_ticks(
                 &blockstore,
@@ -2481,7 +2610,8 @@ pub mod tests {
         blockstore.set_roots(&[3, 5]).unwrap();
 
         // Set up bank1
-        let bank0 = Arc::new(Bank::new(&genesis_config));
+        let mut bank0 = Arc::new(Bank::new(&genesis_config));
+        initiate_callback(&mut bank0, &genesis_config);
         let opts = ProcessOptions {
             poh_verify: true,
             ..ProcessOptions::default()
@@ -2502,13 +2632,14 @@ pub mod tests {
         bank1.squash();
 
         // Test process_blockstore_from_root() from slot 1 onwards
-        let (bank_forks, _leader_schedule) = process_blockstore_from_root(
+        let (bank_forks, _leader_schedule) = do_process_blockstore_from_root(
             &genesis_config,
             &blockstore,
             bank1,
             &opts,
             &recyclers,
             None,
+            false,
         )
         .unwrap();
 
@@ -2824,4 +2955,244 @@ pub mod tests {
             .collect();
         assert_eq!(successes, expected_successful_voter_pubkeys);
     }
+
+    fn make_slot_with_vote_tx(
+        blockstore: &Blockstore,
+        ticks_per_slot: u64,
+        tx_landed_slot: Slot,
+        parent_slot: Slot,
+        parent_blockhash: &Hash,
+        vote_tx: Transaction,
+        slot_leader_keypair: &Arc<Keypair>,
+    ) {
+        // Add votes to `last_slot` so that `root` will be confirmed
+        let vote_entry = next_entry(&parent_blockhash, 1, vec![vote_tx]);
+        let mut entries = create_ticks(ticks_per_slot, 0, vote_entry.hash);
+        entries.insert(0, vote_entry);
+        blockstore
+            .write_entries(
+                tx_landed_slot,
+                0,
+                0,
+                ticks_per_slot,
+                Some(parent_slot),
+                true,
+                &slot_leader_keypair,
+                entries,
+                0,
+            )
+            .unwrap();
+    }
+
+    fn run_test_process_blockstore_with_supermajority_root(blockstore_root: Option<Slot>) {
+        solana_logger::setup();
+        /*
+            Build fork structure:
+                 slot 0
+                   |
+                 slot 1 <- (blockstore root)
+                 /    \
+            slot 2    |
+               |      |
+            slot 4    |
+                    slot 5
+                      |
+                `expected_root_slot`
+                     /    \
+                  ...    minor fork
+                  /
+            `last_slot`
+        */
+        let starting_fork_slot = 5;
+        let mut main_fork = tr(starting_fork_slot);
+        let mut main_fork_ref = main_fork.root_mut();
+
+        // Make enough slots to make a root slot > blockstore_root
+        let expected_root_slot = starting_fork_slot + blockstore_root.unwrap_or(0);
+        let last_main_fork_slot = expected_root_slot + MAX_LOCKOUT_HISTORY as u64 + 1;
+
+        // Make `minor_fork`
+        let last_minor_fork_slot = last_main_fork_slot + 1;
+        let minor_fork = tr(last_minor_fork_slot);
+
+        // Make 'main_fork`
+        for slot in starting_fork_slot + 1..last_main_fork_slot {
+            if slot - 1 == expected_root_slot {
+                main_fork_ref.push_front(minor_fork.clone());
+            }
+            main_fork_ref.push_front(tr(slot));
+            main_fork_ref = main_fork_ref.first_mut().unwrap();
+        }
+        let forks = tr(0) / (tr(1) / (tr(2) / (tr(4))) / main_fork);
+        let validator_keypairs = ValidatorVoteKeypairs::new_rand();
+        let GenesisConfigInfo { genesis_config, .. } =
+            genesis_utils::create_genesis_config_with_vote_accounts(
+                10_000,
+                &[&validator_keypairs],
+                vec![100],
+            );
+        let ticks_per_slot = genesis_config.ticks_per_slot();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Blockstore::open(&ledger_path).unwrap();
+        blockstore.add_tree(forks, false, true, ticks_per_slot, genesis_config.hash());
+
+        if let Some(blockstore_root) = blockstore_root {
+            blockstore.set_roots(&[blockstore_root]).unwrap();
+        }
+
+        let opts = ProcessOptions {
+            poh_verify: true,
+            ..ProcessOptions::default()
+        };
+        let (bank_forks, _leader_schedule) =
+            process_blockstore(&genesis_config, &blockstore, Vec::new(), opts.clone()).unwrap();
+
+        let last_vote_bank_hash = bank_forks.get(last_main_fork_slot - 1).unwrap().hash();
+        let last_vote_blockhash = bank_forks
+            .get(last_main_fork_slot - 1)
+            .unwrap()
+            .last_blockhash();
+        let slots: Vec<_> = (expected_root_slot..last_main_fork_slot).collect();
+        let vote_tx = vote_transaction::new_vote_transaction(
+            slots,
+            last_vote_bank_hash,
+            last_vote_blockhash,
+            &validator_keypairs.node_keypair,
+            &validator_keypairs.vote_keypair,
+            &validator_keypairs.vote_keypair,
+            None,
+        );
+
+        // Add votes to `last_slot` so that `root` will be confirmed
+        let leader_keypair = Arc::new(validator_keypairs.node_keypair);
+        make_slot_with_vote_tx(
+            &blockstore,
+            ticks_per_slot,
+            last_main_fork_slot,
+            last_main_fork_slot - 1,
+            &last_vote_blockhash,
+            vote_tx,
+            &leader_keypair,
+        );
+
+        let (bank_forks, _leader_schedule) =
+            process_blockstore(&genesis_config, &blockstore, Vec::new(), opts).unwrap();
+
+        assert_eq!(bank_forks.root(), expected_root_slot);
+        assert_eq!(
+            bank_forks.frozen_banks().len() as u64,
+            last_minor_fork_slot - expected_root_slot + 1
+        );
+
+        // Minor fork at `last_main_fork_slot + 1` was above the `expected_root_slot`
+        // so should not have been purged
+        //
+        // Fork at slot 2 was purged because it was below the `expected_root_slot`
+        for slot in 0..=last_minor_fork_slot {
+            if slot >= expected_root_slot {
+                let bank = bank_forks.get(slot).unwrap();
+                assert_eq!(bank.slot(), slot);
+                assert!(bank.is_frozen());
+            } else {
+                assert!(bank_forks.get(slot).is_none());
+            }
+        }
+    }
+
+    #[test]
+    fn test_process_blockstore_with_supermajority_root() {
+        run_test_process_blockstore_with_supermajority_root(None);
+        run_test_process_blockstore_with_supermajority_root(Some(1))
+    }
+
+    #[test]
+    fn test_process_blockstore_feature_activations_since_genesis() {
+        let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(123);
+
+        let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config);
+        let blockstore = Blockstore::open(&ledger_path).unwrap();
+
+        let opts = ProcessOptions::default();
+        let (bank_forks, _leader_schedule) =
+            process_blockstore(&genesis_config, &blockstore, vec![], opts).unwrap();
+
+        assert_eq!(bank_forks.working_bank().slot(), 0);
+        assert_eq!(
+            bank_forks.working_bank().builtin_loader_ids(),
+            vec![
+                solana_sdk::bpf_loader::id(),
+                solana_sdk::bpf_loader_deprecated::id()
+            ]
+        );
+    }
+
+    #[test]
+    fn test_process_blockstore_feature_activations_from_snapshot() {
+        let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(123);
+
+        let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config);
+        let blockstore = Blockstore::open(&ledger_path).unwrap();
+
+        // Set up bank1
+        let mut bank0 = Arc::new(Bank::new(&genesis_config));
+        initiate_callback(&mut bank0, &genesis_config);
+        let recyclers = VerifyRecyclers::default();
+        let opts = ProcessOptions::default();
+        process_bank_0(&bank0, &blockstore, &opts, &recyclers).unwrap();
+        let restored_slot = genesis_config.epoch_schedule.get_first_slot_in_epoch(1);
+        let mut bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), restored_slot);
+        bank1.squash();
+
+        // this is similar to snapshot deserialization
+        bank1.reset_callback_and_message_processor();
+        assert_eq!(bank1.builtin_loader_ids(), vec![]);
+
+        let bank1 = Arc::new(bank1);
+        let (bank_forks, _leader_schedule) = process_blockstore_from_root(
+            &genesis_config,
+            &blockstore,
+            bank1,
+            &opts,
+            &recyclers,
+            None,
+        )
+        .unwrap();
+        assert_eq!(bank_forks.working_bank().slot(), restored_slot);
+        assert_eq!(
+            bank_forks.working_bank().builtin_loader_ids(),
+            vec![
+                solana_sdk::bpf_loader::id(),
+                solana_sdk::bpf_loader_deprecated::id()
+            ]
+        );
+    }
+
+    #[test]
+    fn test_process_blockstore_feature_activations_into_epoch_with_activation() {
+        let GenesisConfigInfo {
+            mut genesis_config, ..
+        } = create_genesis_config(123);
+
+        genesis_config.operating_mode = solana_sdk::genesis_config::OperatingMode::Stable;
+        let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config);
+        let blockstore = Blockstore::open(&ledger_path).unwrap();
+
+        let opts = ProcessOptions::default();
+        let (bank_forks, _leader_schedule) =
+            process_blockstore(&genesis_config, &blockstore, vec![], opts).unwrap();
+        let bank0 = bank_forks.working_bank();
+        assert_eq!(bank0.builtin_loader_ids(), vec![]);
+
+        let restored_slot = genesis_config.epoch_schedule.get_first_slot_in_epoch(34);
+        let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), restored_slot);
+
+        assert_eq!(bank0.slot(), 0);
+        assert_eq!(bank0.builtin_loader_ids(), vec![]);
+
+        assert_eq!(bank1.slot(), restored_slot);
+        assert_eq!(
+            bank1.builtin_loader_ids(),
+            vec![solana_sdk::bpf_loader_deprecated::id()]
+        );
+    }
 }

local-cluster/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-local-cluster"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,22 +13,22 @@ itertools = "0.9.0"
 gag = "0.1.10"
 log = "0.4.8"
 rand = "0.7.0"
-solana-config-program = { path = "../programs/config", version = "1.3.4" }
-solana-core = { path = "../core", version = "1.3.4" }
-solana-client = { path = "../client", version = "1.3.4" }
-solana-download-utils = { path = "../download-utils", version = "1.3.4" }
-solana-faucet = { path = "../faucet", version = "1.3.4" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.3.4" }
-solana-genesis-programs = { path = "../genesis-programs", version = "1.3.4" }
-solana-ledger = { path = "../ledger", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-runtime = { path = "../runtime", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-stake-program = { path = "../programs/stake", version = "1.3.4" }
-solana-vest-program = { path = "../programs/vest", version = "1.3.4" }
-solana-vote-program = { path = "../programs/vote", version = "1.3.4" }
+solana-config-program = { path = "../programs/config", version = "1.3.6" }
+solana-core = { path = "../core", version = "1.3.6" }
+solana-client = { path = "../client", version = "1.3.6" }
+solana-download-utils = { path = "../download-utils", version = "1.3.6" }
+solana-faucet = { path = "../faucet", version = "1.3.6" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.3.6" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.3.6" }
+solana-ledger = { path = "../ledger", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-runtime = { path = "../runtime", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-stake-program = { path = "../programs/stake", version = "1.3.6" }
+solana-vest-program = { path = "../programs/vest", version = "1.3.6" }
+solana-vote-program = { path = "../programs/vote", version = "1.3.6" }
 tempfile = "3.1.0"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.3.4" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.3.6" }
 
 [dev-dependencies]
 assert_matches = "1.3.0"

local-cluster/src/local_cluster.rs

@@ -135,9 +135,10 @@ impl LocalCluster {
                     if *in_genesis {
                         Some((
                             ValidatorVoteKeypairs {
-                                node_keypair: node_keypair.clone(),
-                                vote_keypair: Arc::new(Keypair::new()),
-                                stake_keypair: Arc::new(Keypair::new()),
+                                node_keypair: Keypair::from_bytes(&node_keypair.to_bytes())
+                                    .unwrap(),
+                                vote_keypair: Keypair::new(),
+                                stake_keypair: Keypair::new(),
                             },
                             stake,
                         ))
@@ -171,8 +172,9 @@ impl LocalCluster {
         match genesis_config.operating_mode {
             OperatingMode::Stable | OperatingMode::Preview => {
                 genesis_config.native_instruction_processors =
-                    solana_genesis_programs::get_programs(genesis_config.operating_mode, 0)
-                        .unwrap_or_default()
+                    solana_genesis_programs::get_native_programs_for_genesis(
+                        genesis_config.operating_mode,
+                    )
             }
             _ => (),
         }
@@ -202,11 +204,15 @@ impl LocalCluster {
         leader_config.rpc_ports = Some((
             leader_node.info.rpc.port(),
             leader_node.info.rpc_pubsub.port(),
+            leader_node.info.rpc_banks.port(),
         ));
         leader_config.account_paths = vec![leader_ledger_path.join("accounts")];
+        let leader_keypair = Arc::new(Keypair::from_bytes(&leader_keypair.to_bytes()).unwrap());
+        let leader_vote_keypair =
+            Arc::new(Keypair::from_bytes(&leader_vote_keypair.to_bytes()).unwrap());
         let leader_server = Validator::new(
             leader_node,
-            leader_keypair,
+            &leader_keypair,
             &leader_ledger_path,
             &leader_vote_keypair.pubkey(),
             vec![leader_vote_keypair.clone()],
@@ -217,8 +223,8 @@ impl LocalCluster {
 
         let mut validators = HashMap::new();
         let leader_info = ValidatorInfo {
-            keypair: leader_keypair.clone(),
-            voting_keypair: leader_vote_keypair.clone(),
+            keypair: leader_keypair,
+            voting_keypair: leader_vote_keypair,
             ledger_path: leader_ledger_path,
             contact_info: leader_contact_info.clone(),
         };
@@ -240,7 +246,12 @@ impl LocalCluster {
 
         let node_pubkey_to_vote_key: HashMap<Pubkey, Arc<Keypair>> = keys_in_genesis
             .into_iter()
-            .map(|keypairs| (keypairs.node_keypair.pubkey(), keypairs.vote_keypair))
+            .map(|keypairs| {
+                (
+                    keypairs.node_keypair.pubkey(),
+                    Arc::new(Keypair::from_bytes(&keypairs.vote_keypair.to_bytes()).unwrap()),
+                )
+            })
             .collect();
         for (stake, validator_config, (key, _)) in izip!(
             (&config.node_stakes[1..]).iter(),
@@ -343,6 +354,7 @@ impl LocalCluster {
         config.rpc_ports = Some((
             validator_node.info.rpc.port(),
             validator_node.info.rpc_pubsub.port(),
+            validator_node.info.rpc_banks.port(),
         ));
         config.account_paths = vec![ledger_path.join("accounts")];
         let voting_keypair = voting_keypair.unwrap();
@@ -613,8 +625,11 @@ impl Cluster for LocalCluster {
         // Update the stored ContactInfo for this node
         let node = Node::new_localhost_with_pubkey(&pubkey);
         cluster_validator_info.info.contact_info = node.info.clone();
-        cluster_validator_info.config.rpc_ports =
-            Some((node.info.rpc.port(), node.info.rpc_pubsub.port()));
+        cluster_validator_info.config.rpc_ports = Some((
+            node.info.rpc.port(),
+            node.info.rpc_pubsub.port(),
+            node.info.rpc_banks.port(),
+        ));
 
         let entry_point_info = {
             if *pubkey == self.entry_point_info.id {

local-cluster/tests/local_cluster.rs

@@ -705,7 +705,13 @@ fn test_stable_operating_mode() {
     }
 
     // Programs that are not available at epoch 0
-    for program_id in [&solana_sdk::bpf_loader::id(), &solana_vest_program::id()].iter() {
+    for program_id in [
+        &solana_sdk::bpf_loader::id(),
+        &solana_sdk::bpf_loader_deprecated::id(),
+        &solana_vest_program::id(),
+    ]
+    .iter()
+    {
         assert_eq!(
             (
                 program_id,

log-analyzer/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-log-analyzer"
 description = "The solana cluster network analysis tool"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,9 +14,9 @@ byte-unit = "4.0.8"
 clap = "2.33.1"
 serde = "1.0.112"
 serde_json = "1.0.56"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
 
 [[bin]]
 name = "solana-log-analyzer"

logger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-logger"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana Logger"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

measure/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-measure"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.3.4"
+version = "1.3.6"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -12,8 +12,8 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-metrics = { path = "../metrics", version = "1.3.4" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-metrics = { path = "../metrics", version = "1.3.6" }
 
 [target."cfg(unix)".dependencies]
 jemallocator = "0.3.2"

merkle-tree/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-merkle-tree"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana Merkle Tree"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../sdk", version = "1.3.4" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
 fast-math = "0.1"
 
 [dev-dependencies]

metrics/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-metrics"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana Metrics"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,7 +14,7 @@ gethostname = "0.2.1"
 lazy_static = "1.4.0"
 log = "0.4.8"
 reqwest = { version = "0.10.6", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
 
 [dev-dependencies]
 rand = "0.7.0"

net-shaper/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-net-shaper"
 description = "The solana cluster network shaping tool"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,8 +13,8 @@ publish = false
 clap = "2.33.1"
 serde = "1.0.112"
 serde_json = "1.0.56"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
 rand = "0.7.0"
 
 [[bin]]

net-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-net-utils"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana Network Utilities"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -18,9 +18,9 @@ rand = "0.7.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 socket2 = "0.3.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
 tokio = "0.1"
 tokio-codec = "0.1"
 url = "2.1.1"

notifier/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-notifier"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana Notifier"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

perf/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-perf"
-version = "1.3.4"
+version = "1.3.6"
 description = "Solana Performance APIs"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -17,11 +17,11 @@ serde = "1.0.112"
 dlopen_derive = "0.1.4"
 lazy_static = "1.4.0"
 log = "0.4.8"
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.3.4" }
-solana-budget-program = { path = "../programs/budget", version = "1.3.4" }
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-metrics = { path = "../metrics", version = "1.3.4" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.3.6" }
+solana-budget-program = { path = "../programs/budget", version = "1.3.6" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-metrics = { path = "../metrics", version = "1.3.6" }
 curve25519-dalek = { version = "2" }
 
 [lib]

poh-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-poh-bench"
-version = "1.3.4"
+version = "1.3.6"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -12,13 +12,13 @@ clap = "2.33.1"
 log = "0.4.6"
 rand = "0.7.0"
 rayon = "1.3.0"
-solana-logger = { path = "../logger", version = "1.3.4" }
-solana-ledger = { path = "../ledger", version = "1.3.4" }
-solana-sdk = { path = "../sdk", version = "1.3.4" }
-solana-clap-utils = { path = "../clap-utils", version = "1.3.4" }
-solana-measure = { path = "../measure", version = "1.3.4" }
-solana-version = { path = "../version", version = "1.3.4" }
-solana-perf = { path = "../perf", version = "1.3.4" }
+solana-logger = { path = "../logger", version = "1.3.6" }
+solana-ledger = { path = "../ledger", version = "1.3.6" }
+solana-sdk = { path = "../sdk", version = "1.3.6" }
+solana-clap-utils = { path = "../clap-utils", version = "1.3.6" }
+solana-measure = { path = "../measure", version = "1.3.6" }
+solana-version = { path = "../version", version = "1.3.6" }
+solana-perf = { path = "../perf", version = "1.3.6" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

programs/bpf/Cargo.lock

@@ -1612,11 +1612,10 @@ checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4"
 
 [[package]]
 name = "solana-bpf-loader-program"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "bincode",
  "byteorder 1.3.4",
- "jemalloc-sys",
  "num-derive 0.3.0",
  "num-traits",
  "solana-runtime",
@@ -1627,7 +1626,7 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-programs"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "bincode",
  "byteorder 1.3.4",
@@ -1642,7 +1641,7 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-128bit"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-bpf-rust-128bit-dep",
  "solana-sdk",
@@ -1650,21 +1649,21 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-128bit-dep"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-sdk",
 ]
 
 [[package]]
 name = "solana-bpf-rust-alloc"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-sdk",
 ]
 
 [[package]]
 name = "solana-bpf-rust-dep-crate"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "byteorder 1.3.4",
  "solana-sdk",
@@ -1672,14 +1671,14 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-dup-accounts"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-sdk",
 ]
 
 [[package]]
 name = "solana-bpf-rust-error-handling"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "num-derive 0.2.5",
  "num-traits",
@@ -1689,14 +1688,14 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-external-spend"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-sdk",
 ]
 
 [[package]]
 name = "solana-bpf-rust-invoke"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-bpf-rust-invoked",
  "solana-sdk",
@@ -1704,21 +1703,21 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-invoked"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-sdk",
 ]
 
 [[package]]
 name = "solana-bpf-rust-iter"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-sdk",
 ]
 
 [[package]]
 name = "solana-bpf-rust-many-args"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-bpf-rust-many-args-dep",
  "solana-sdk",
@@ -1726,28 +1725,28 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-many-args-dep"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-sdk",
 ]
 
 [[package]]
 name = "solana-bpf-rust-noop"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-sdk",
 ]
 
 [[package]]
 name = "solana-bpf-rust-panic"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-sdk",
 ]
 
 [[package]]
 name = "solana-bpf-rust-param-passing"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-bpf-rust-param-passing-dep",
  "solana-sdk",
@@ -1755,21 +1754,21 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-param-passing-dep"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-sdk",
 ]
 
 [[package]]
 name = "solana-bpf-rust-sysval"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "solana-sdk",
 ]
 
 [[package]]
 name = "solana-config-program"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "bincode",
  "chrono",
@@ -1781,7 +1780,7 @@ dependencies = [
 
 [[package]]
 name = "solana-crate-features"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "backtrace",
  "bytes 0.4.12",
@@ -1804,7 +1803,7 @@ dependencies = [
 
 [[package]]
 name = "solana-logger"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "env_logger",
  "lazy_static",
@@ -1813,7 +1812,7 @@ dependencies = [
 
 [[package]]
 name = "solana-measure"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "jemalloc-ctl",
  "jemallocator",
@@ -1824,7 +1823,7 @@ dependencies = [
 
 [[package]]
 name = "solana-metrics"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "env_logger",
  "gethostname",
@@ -1836,7 +1835,7 @@ dependencies = [
 
 [[package]]
 name = "solana-rayon-threadlimit"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "lazy_static",
  "num_cpus",
@@ -1844,7 +1843,7 @@ dependencies = [
 
 [[package]]
 name = "solana-runtime"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "bincode",
  "bv",
@@ -1888,7 +1887,7 @@ dependencies = [
 
 [[package]]
 name = "solana-sdk"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "assert_matches",
  "bincode",
@@ -1925,7 +1924,7 @@ dependencies = [
 
 [[package]]
 name = "solana-sdk-macro"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "bs58",
  "proc-macro2 1.0.19",
@@ -1936,7 +1935,7 @@ dependencies = [
 
 [[package]]
 name = "solana-sdk-macro-frozen-abi"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "lazy_static",
  "proc-macro2 1.0.19",
@@ -1947,7 +1946,7 @@ dependencies = [
 
 [[package]]
 name = "solana-stake-program"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "bincode",
  "log",
@@ -1966,7 +1965,7 @@ dependencies = [
 
 [[package]]
 name = "solana-vote-program"
-version = "1.3.4"
+version = "1.3.6"
 dependencies = [
  "bincode",
  "log",
@@ -1984,9 +1983,9 @@ dependencies = [
 
 [[package]]
 name = "solana_rbpf"
-version = "0.1.28"
+version = "0.1.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a556eca8a56761a16d712ed3e62a420da220f43749237befac3e4bf820f939c"
+checksum = "185f68b54660652e2244bbdef792b369f12045da856a4af75b776e6e72757831"
 dependencies = [
  "byteorder 1.3.4",
  "combine",

programs/bpf/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-bpf-programs"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.3.4"
+version = "1.3.6"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "README.md"
@@ -22,11 +22,11 @@ walkdir = "2"
 bincode = "1.1.4"
 byteorder = "1.3.2"
 elf = "0.0.10"
-solana-bpf-loader-program = { path = "../bpf_loader", version = "1.3.4" }
-solana-logger = { path = "../../logger", version = "1.3.4" }
-solana-runtime = { path = "../../runtime", version = "1.3.4" }
-solana-sdk = { path = "../../sdk", version = "1.3.4" }
-solana_rbpf = "=0.1.28"
+solana-bpf-loader-program = { path = "../bpf_loader", version = "1.3.6" }
+solana-logger = { path = "../../logger", version = "1.3.6" }
+solana-runtime = { path = "../../runtime", version = "1.3.6" }
+solana-sdk = { path = "../../sdk", version = "1.3.6" }
+solana_rbpf = "=0.1.30"
 
 [[bench]]
 name = "bpf_loader"

programs/bpf/benches/bpf_loader.rs

@@ -6,7 +6,8 @@ use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
 use solana_rbpf::EbpfVm;
 use solana_sdk::{
     account::Account,
-    entrypoint_native::{InvokeContext, Logger, ProcessInstruction},
+    bpf_loader,
+    entrypoint_native::{ComputeBudget, ComputeMeter, InvokeContext, Logger, ProcessInstruction},
     instruction::{CompiledInstruction, InstructionError},
     message::Message,
     pubkey::Pubkey,
@@ -80,7 +81,9 @@ fn bench_program_alu(bencher: &mut Bencher) {
     let mut invoke_context = MockInvokeContext::default();
 
     let elf = load_elf().unwrap();
-    let (mut vm, _) = solana_bpf_loader_program::create_vm(&elf, &[], &mut invoke_context).unwrap();
+    let (mut vm, _) =
+        solana_bpf_loader_program::create_vm(&bpf_loader::id(), &elf, &[], &mut invoke_context)
+            .unwrap();
 
     println!("Interpreted:");
     assert_eq!(
@@ -96,7 +99,7 @@ fn bench_program_alu(bencher: &mut Bencher) {
     bencher.iter(|| {
         vm.execute_program(&mut inner_iter, &[], &[]).unwrap();
     });
-    let instructions = vm.get_last_instruction_count();
+    let instructions = vm.get_total_instruction_count();
     let summary = bencher.bench(|_bencher| {}).unwrap();
     println!("  {:?} instructions", instructions);
     println!("  {:?} ns/iter median", summary.median as u64);
@@ -136,6 +139,7 @@ fn bench_program_alu(bencher: &mut Bencher) {
 pub struct MockInvokeContext {
     key: Pubkey,
     mock_logger: MockLogger,
+    mock_compute_meter: MockComputeMeter,
 }
 impl InvokeContext for MockInvokeContext {
     fn push(&mut self, _key: &Pubkey) -> Result<(), InstructionError> {
@@ -162,6 +166,12 @@ impl InvokeContext for MockInvokeContext {
     fn is_cross_program_supported(&self) -> bool {
         true
     }
+    fn get_compute_budget(&self) -> ComputeBudget {
+        ComputeBudget::default()
+    }
+    fn get_compute_meter(&self) -> Rc<RefCell<dyn ComputeMeter>> {
+        Rc::new(RefCell::new(self.mock_compute_meter.clone()))
+    }
 }
 #[derive(Debug, Default, Clone)]
 pub struct MockLogger {
@@ -175,3 +185,19 @@ impl Logger for MockLogger {
         self.log.borrow_mut().push(message.to_string());
     }
 }
+#[derive(Debug, Default, Clone)]
+pub struct MockComputeMeter {
+    pub remaining: u64,
+}
+impl ComputeMeter for MockComputeMeter {
+    fn consume(&mut self, amount: u64) -> Result<(), InstructionError> {
+        self.remaining = self.remaining.saturating_sub(amount);
+        if self.remaining == 0 {
+            return Err(InstructionError::ComputationalBudgetExceeded);
+        }
+        Ok(())
+    }
+    fn get_remaining(&self) -> u64 {
+        self.remaining
+    }
+}

programs/bpf/c/src/invoke/invoke.c

@@ -36,21 +36,62 @@ extern uint64_t entrypoint(const uint8_t *input) {
 
   switch (params.data[0]) {
   case TEST_SUCCESS: {
-    sol_log("Call system program");
+    sol_log("Call system program create account");
     {
-      sol_assert(*accounts[FROM_INDEX].lamports = 43);
-      sol_assert(*accounts[ARGUMENT_INDEX].lamports = 41);
+      uint64_t from_lamports = *accounts[FROM_INDEX].lamports;
+      uint64_t to_lamports = *accounts[DERIVED_KEY1_INDEX].lamports;
       SolAccountMeta arguments[] = {
-          {accounts[FROM_INDEX].key, false, true},
-          {accounts[ARGUMENT_INDEX].key, false, false}};
+          {accounts[FROM_INDEX].key, true, true},
+          {accounts[DERIVED_KEY1_INDEX].key, true, true}};
+      uint8_t data[4 + 8 + 8 + 32];
+      *(uint64_t *)(data + 4) = 42;
+      *(uint64_t *)(data + 4 + 8) = MAX_PERMITTED_DATA_INCREASE;
+      sol_memcpy(data + 4 + 8 + 8, params.program_id, SIZE_PUBKEY);
+      const SolInstruction instruction = {accounts[SYSTEM_PROGRAM_INDEX].key,
+                                          arguments, SOL_ARRAY_SIZE(arguments),
+                                          data, SOL_ARRAY_SIZE(data)};
+      uint8_t seed1[] = {'Y', 'o', 'u', ' ', 'p', 'a', 's', 's',
+                         ' ', 'b', 'u', 't', 't', 'e', 'r'};
+      const SolSignerSeed seeds1[] = {{seed1, SOL_ARRAY_SIZE(seed1)},
+                                      {&nonce1, 1}};
+      const SolSignerSeeds signers_seeds[] = {{seeds1, SOL_ARRAY_SIZE(seeds1)}};
+      sol_assert(SUCCESS == sol_invoke_signed(&instruction, accounts,
+                                              SOL_ARRAY_SIZE(accounts),
+                                              signers_seeds,
+                                              SOL_ARRAY_SIZE(signers_seeds)));
+      sol_assert(*accounts[FROM_INDEX].lamports == from_lamports - 42);
+      sol_assert(*accounts[DERIVED_KEY1_INDEX].lamports == to_lamports + 42);
+      sol_assert(SolPubkey_same(accounts[DERIVED_KEY1_INDEX].owner,
+                                params.program_id));
+      sol_assert(accounts[DERIVED_KEY1_INDEX].data_len ==
+                 MAX_PERMITTED_DATA_INCREASE);
+      sol_assert(
+          accounts[DERIVED_KEY1_INDEX].data[MAX_PERMITTED_DATA_INCREASE - 1] ==
+          0);
+      accounts[DERIVED_KEY1_INDEX].data[MAX_PERMITTED_DATA_INCREASE - 1] = 0x0f;
+      sol_assert(
+          accounts[DERIVED_KEY1_INDEX].data[MAX_PERMITTED_DATA_INCREASE - 1] ==
+          0x0f);
+      for (uint8_t i = 0; i < 20; i++) {
+        accounts[DERIVED_KEY1_INDEX].data[i] = i;
+      }
+    }
+
+    sol_log("Call system program transfer");
+    {
+      uint64_t from_lamports = *accounts[FROM_INDEX].lamports;
+      uint64_t to_lamports = *accounts[DERIVED_KEY1_INDEX].lamports;
+      SolAccountMeta arguments[] = {
+          {accounts[FROM_INDEX].key, true, true},
+          {accounts[DERIVED_KEY1_INDEX].key, true, false}};
       uint8_t data[] = {2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0};
       const SolInstruction instruction = {accounts[SYSTEM_PROGRAM_INDEX].key,
                                           arguments, SOL_ARRAY_SIZE(arguments),
                                           data, SOL_ARRAY_SIZE(data)};
       sol_assert(SUCCESS ==
                  sol_invoke(&instruction, accounts, SOL_ARRAY_SIZE(accounts)));
-      sol_assert(*accounts[FROM_INDEX].lamports = 42);
-      sol_assert(*accounts[ARGUMENT_INDEX].lamports = 42);
+      sol_assert(*accounts[FROM_INDEX].lamports == from_lamports - 1);
+      sol_assert(*accounts[DERIVED_KEY1_INDEX].lamports == to_lamports + 1);
     }
 
     sol_log("Test data translation");
@@ -92,8 +133,9 @@ extern uint64_t entrypoint(const uint8_t *input) {
       const SolSignerSeed seeds1[] = {{seed1, SOL_ARRAY_SIZE(seed1)},
                                       {&nonce1, 1}};
       SolPubkey address;
-      sol_assert(SUCCESS == sol_create_program_address(seeds1, SOL_ARRAY_SIZE(seeds1),
-                                                 params.program_id, &address));
+      sol_assert(SUCCESS ==
+                 sol_create_program_address(seeds1, SOL_ARRAY_SIZE(seeds1),
+                                            params.program_id, &address));
       sol_assert(SolPubkey_same(&address, accounts[DERIVED_KEY1_INDEX].key));
     }
 
@@ -123,6 +165,33 @@ extern uint64_t entrypoint(const uint8_t *input) {
                                               SOL_ARRAY_SIZE(signers_seeds)));
     }
 
+    sol_log("Test multiple derived signers");
+    {
+      SolAccountMeta arguments[] = {
+          {accounts[DERIVED_KEY1_INDEX].key, true, false},
+          {accounts[DERIVED_KEY2_INDEX].key, true, true},
+          {accounts[DERIVED_KEY3_INDEX].key, false, true}};
+      uint8_t data[] = {TEST_VERIFY_NESTED_SIGNERS};
+      const SolInstruction instruction = {accounts[INVOKED_PROGRAM_INDEX].key,
+                                          arguments, SOL_ARRAY_SIZE(arguments),
+                                          data, SOL_ARRAY_SIZE(data)};
+      uint8_t seed1[] = {'L', 'i', 'l', '\''};
+      uint8_t seed2[] = {'B', 'i', 't', 's'};
+      const SolSignerSeed seeds1[] = {{seed1, SOL_ARRAY_SIZE(seed1)},
+                                      {seed2, SOL_ARRAY_SIZE(seed2)},
+                                      {&nonce2, 1}};
+      const SolSignerSeed seeds2[] = {
+          {(uint8_t *)accounts[DERIVED_KEY2_INDEX].key, SIZE_PUBKEY},
+          {&nonce3, 1}};
+      const SolSignerSeeds signers_seeds[] = {{seeds1, SOL_ARRAY_SIZE(seeds1)},
+                                              {seeds2, SOL_ARRAY_SIZE(seeds2)}};
+
+      sol_assert(SUCCESS == sol_invoke_signed(&instruction, accounts,
+                                              SOL_ARRAY_SIZE(accounts),
+                                              signers_seeds,
+                                              SOL_ARRAY_SIZE(signers_seeds)));
+    }
+
     sol_log("Test readonly with writable account");
     {
       SolAccountMeta arguments[] = {

programs/bpf/c/src/invoked/invoked.c

@@ -23,6 +23,11 @@ extern uint64_t entrypoint(const uint8_t *input) {
     sol_assert(sol_deserialize(input, &params, 4));
 
     SolPubkey bpf_loader_id =
+        (SolPubkey){.x = {2,  168, 246, 145, 78,  136, 161, 110, 57,  90, 225,
+                          40, 148, 143, 250, 105, 86,  147, 55,  104, 24, 221,
+                          71, 67,  82,  33,  243, 198, 0,   0,   0,   0}};
+
+    SolPubkey bpf_loader_deprecated_id =
         (SolPubkey){.x = {2,   168, 246, 145, 78,  136, 161, 107, 189, 35,  149,
                           133, 95,  100, 4,   217, 180, 244, 86,  183, 130, 27,
                           176, 20,  87,  73,  66,  140, 0,   0,   0,   0}};
@@ -92,31 +97,6 @@ extern uint64_t entrypoint(const uint8_t *input) {
     sol_assert(!accounts[DERIVED_KEY2_INDEX].is_signer);
     sol_assert(!accounts[DERIVED_KEY2_INDEX].is_signer);
 
-    uint8_t nonce2 = params.data[1];
-    uint8_t nonce3 = params.data[2];
-
-    SolAccountMeta arguments[] = {
-        {accounts[DERIVED_KEY1_INDEX].key, true, false},
-        {accounts[DERIVED_KEY2_INDEX].key, true, true},
-        {accounts[DERIVED_KEY3_INDEX].key, false, true}};
-    uint8_t data[] = {TEST_VERIFY_NESTED_SIGNERS};
-    const SolInstruction instruction = {accounts[INVOKED_PROGRAM_INDEX].key,
-                                        arguments, SOL_ARRAY_SIZE(arguments),
-                                        data, SOL_ARRAY_SIZE(data)};
-    uint8_t seed1[] = {'L', 'i', 'l', '\''};
-    uint8_t seed2[] = {'B', 'i', 't', 's'};
-    const SolSignerSeed seeds1[] = {{seed1, SOL_ARRAY_SIZE(seed1)},
-                                    {seed2, SOL_ARRAY_SIZE(seed2)},
-                                    {&nonce2, 1}};
-    const SolSignerSeed seeds2[] = {
-        {(uint8_t *)accounts[DERIVED_KEY2_INDEX].key, SIZE_PUBKEY},
-        {&nonce3, 1}};
-    const SolSignerSeeds signers_seeds[] = {{seeds1, SOL_ARRAY_SIZE(seeds1)},
-                                            {seeds2, SOL_ARRAY_SIZE(seeds2)}};
-
-    sol_assert(SUCCESS == sol_invoke_signed(
-                              &instruction, accounts, SOL_ARRAY_SIZE(accounts),
-                              signers_seeds, SOL_ARRAY_SIZE(signers_seeds)));
     break;
   }