Fully enable cross program support in mainnet-beta

This reverts commit d6ca879d39.

Cargo.toml

@@ -25,6 +25,7 @@ members = [
     "log-analyzer",
     "merkle-tree",
     "stake-o-matic",
+    "storage-bigtable",
     "streamer",
     "measure",
     "metrics",

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,11 +10,15 @@ edition = "2018"
 
 [dependencies]
 bincode = "1.2.1"
+base64 = "0.12.3"
 bs58 = "0.3.1"
+bv = "0.11.1"
 Inflector = "0.11.4"
 lazy_static = "1.4.0"
-solana-sdk = { path = "../sdk", version = "1.2.19"  }
-solana-vote-program = { path = "../programs/vote", version = "1.2.19" }
+solana-config-program = { path = "../programs/config", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.24" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.24" }
 spl-token-v1-0 = { package = "spl-token", version = "1.0.6", features = ["skip-no-mangle"] }
 serde = "1.0.112"
 serde_derive = "1.0.103"

account-decoder/src/lib.rs

@@ -4,14 +4,20 @@ extern crate lazy_static;
 extern crate serde_derive;
 
 pub mod parse_account_data;
+pub mod parse_config;
 pub mod parse_nonce;
+pub mod parse_stake;
+pub mod parse_sysvar;
 pub mod parse_token;
 pub mod parse_vote;
+pub mod validator_info;
 
-use crate::parse_account_data::{parse_account_data, ParsedAccount};
-use solana_sdk::{account::Account, clock::Epoch, pubkey::Pubkey};
+use crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount};
+use solana_sdk::{account::Account, clock::Epoch, fee_calculator::FeeCalculator, pubkey::Pubkey};
 use std::str::FromStr;
 
+pub type StringAmount = String;
+
 /// A duplicate representation of an Account for pretty JSON serialization
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
@@ -28,30 +34,40 @@ pub struct UiAccount {
 pub enum UiAccountData {
     Binary(String),
     Json(ParsedAccount),
-}
-
-impl From<Vec<u8>> for UiAccountData {
-    fn from(data: Vec<u8>) -> Self {
-        Self::Binary(bs58::encode(data).into_string())
-    }
+    Binary64(String),
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub enum UiAccountEncoding {
-    Binary,
+    Binary, // SLOW! Avoid this encoding
     JsonParsed,
+    Binary64,
 }
 
 impl UiAccount {
-    pub fn encode(account: Account, encoding: UiAccountEncoding) -> Self {
+    pub fn encode(
+        pubkey: &Pubkey,
+        account: Account,
+        encoding: UiAccountEncoding,
+        additional_data: Option<AccountAdditionalData>,
+        data_slice_config: Option<UiDataSliceConfig>,
+    ) -> Self {
         let data = match encoding {
-            UiAccountEncoding::Binary => account.data.into(),
+            UiAccountEncoding::Binary => UiAccountData::Binary(
+                bs58::encode(slice_data(&account.data, data_slice_config)).into_string(),
+            ),
+            UiAccountEncoding::Binary64 => UiAccountData::Binary64(base64::encode(slice_data(
+                &account.data,
+                data_slice_config,
+            ))),
             UiAccountEncoding::JsonParsed => {
-                if let Ok(parsed_data) = parse_account_data(&account.owner, &account.data) {
+                if let Ok(parsed_data) =
+                    parse_account_data(pubkey, &account.owner, &account.data, additional_data)
+                {
                     UiAccountData::Json(parsed_data)
                 } else {
-                    account.data.into()
+                    UiAccountData::Binary64(base64::encode(&account.data))
                 }
             }
         };
@@ -68,6 +84,7 @@ impl UiAccount {
         let data = match &self.data {
             UiAccountData::Json(_) => None,
             UiAccountData::Binary(blob) => bs58::decode(blob).into_vec().ok(),
+            UiAccountData::Binary64(blob) => base64::decode(blob).ok(),
         }?;
         Some(Account {
             lamports: self.lamports,
@@ -78,3 +95,79 @@ impl UiAccount {
         })
     }
 }
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiFeeCalculator {
+    pub lamports_per_signature: StringAmount,
+}
+
+impl From<FeeCalculator> for UiFeeCalculator {
+    fn from(fee_calculator: FeeCalculator) -> Self {
+        Self {
+            lamports_per_signature: fee_calculator.lamports_per_signature.to_string(),
+        }
+    }
+}
+
+impl Default for UiFeeCalculator {
+    fn default() -> Self {
+        Self {
+            lamports_per_signature: "0".to_string(),
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct UiDataSliceConfig {
+    pub offset: usize,
+    pub length: usize,
+}
+
+fn slice_data(data: &[u8], data_slice_config: Option<UiDataSliceConfig>) -> &[u8] {
+    if let Some(UiDataSliceConfig { offset, length }) = data_slice_config {
+        if offset >= data.len() {
+            &[]
+        } else if length > data.len() - offset {
+            &data[offset..]
+        } else {
+            &data[offset..offset + length]
+        }
+    } else {
+        data
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_slice_data() {
+        let data = vec![1, 2, 3, 4, 5];
+        let slice_config = Some(UiDataSliceConfig {
+            offset: 0,
+            length: 5,
+        });
+        assert_eq!(slice_data(&data, slice_config), &data[..]);
+
+        let slice_config = Some(UiDataSliceConfig {
+            offset: 0,
+            length: 10,
+        });
+        assert_eq!(slice_data(&data, slice_config), &data[..]);
+
+        let slice_config = Some(UiDataSliceConfig {
+            offset: 1,
+            length: 2,
+        });
+        assert_eq!(slice_data(&data, slice_config), &data[1..3]);
+
+        let slice_config = Some(UiDataSliceConfig {
+            offset: 10,
+            length: 2,
+        });
+        assert_eq!(slice_data(&data, slice_config), &[] as &[u8]);
+    }
+}

account-decoder/src/parse_account_data.rs

@@ -1,22 +1,31 @@
 use crate::{
+    parse_config::parse_config,
     parse_nonce::parse_nonce,
+    parse_stake::parse_stake,
+    parse_sysvar::parse_sysvar,
     parse_token::{parse_token, spl_token_id_v1_0},
     parse_vote::parse_vote,
 };
 use inflector::Inflector;
 use serde_json::Value;
-use solana_sdk::{instruction::InstructionError, pubkey::Pubkey, system_program};
+use solana_sdk::{instruction::InstructionError, pubkey::Pubkey, system_program, sysvar};
 use std::collections::HashMap;
 use thiserror::Error;
 
 lazy_static! {
+    static ref CONFIG_PROGRAM_ID: Pubkey = solana_config_program::id();
+    static ref STAKE_PROGRAM_ID: Pubkey = solana_stake_program::id();
     static ref SYSTEM_PROGRAM_ID: Pubkey = system_program::id();
+    static ref SYSVAR_PROGRAM_ID: Pubkey = sysvar::id();
     static ref TOKEN_PROGRAM_ID: Pubkey = spl_token_id_v1_0();
     static ref VOTE_PROGRAM_ID: Pubkey = solana_vote_program::id();
     pub static ref PARSABLE_PROGRAM_IDS: HashMap<Pubkey, ParsableAccount> = {
         let mut m = HashMap::new();
+        m.insert(*CONFIG_PROGRAM_ID, ParsableAccount::Config);
         m.insert(*SYSTEM_PROGRAM_ID, ParsableAccount::Nonce);
         m.insert(*TOKEN_PROGRAM_ID, ParsableAccount::SplToken);
+        m.insert(*STAKE_PROGRAM_ID, ParsableAccount::Stake);
+        m.insert(*SYSVAR_PROGRAM_ID, ParsableAccount::Sysvar);
         m.insert(*VOTE_PROGRAM_ID, ParsableAccount::Vote);
         m
     };
@@ -30,6 +39,9 @@ pub enum ParseAccountError {
     #[error("Program not parsable")]
     ProgramNotParsable,
 
+    #[error("Additional data required to parse: {0}")]
+    AdditionalDataMissing(String),
+
     #[error("Instruction error")]
     InstructionError(#[from] InstructionError),
 
@@ -42,31 +54,49 @@ pub enum ParseAccountError {
 pub struct ParsedAccount {
     pub program: String,
     pub parsed: Value,
+    pub space: u64,
 }
 
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum ParsableAccount {
+    Config,
     Nonce,
     SplToken,
+    Stake,
+    Sysvar,
     Vote,
 }
 
+#[derive(Default)]
+pub struct AccountAdditionalData {
+    pub spl_token_decimals: Option<u8>,
+}
+
 pub fn parse_account_data(
+    pubkey: &Pubkey,
     program_id: &Pubkey,
     data: &[u8],
+    additional_data: Option<AccountAdditionalData>,
 ) -> Result<ParsedAccount, ParseAccountError> {
     let program_name = PARSABLE_PROGRAM_IDS
         .get(program_id)
         .ok_or_else(|| ParseAccountError::ProgramNotParsable)?;
+    let additional_data = additional_data.unwrap_or_default();
     let parsed_json = match program_name {
+        ParsableAccount::Config => serde_json::to_value(parse_config(data, pubkey)?)?,
         ParsableAccount::Nonce => serde_json::to_value(parse_nonce(data)?)?,
-        ParsableAccount::SplToken => serde_json::to_value(parse_token(data)?)?,
+        ParsableAccount::SplToken => {
+            serde_json::to_value(parse_token(data, additional_data.spl_token_decimals)?)?
+        }
+        ParsableAccount::Stake => serde_json::to_value(parse_stake(data)?)?,
+        ParsableAccount::Sysvar => serde_json::to_value(parse_sysvar(data, pubkey)?)?,
         ParsableAccount::Vote => serde_json::to_value(parse_vote(data)?)?,
     };
     Ok(ParsedAccount {
         program: format!("{:?}", program_name).to_kebab_case(),
         parsed: parsed_json,
+        space: data.len() as u64,
     })
 }
 
@@ -81,20 +111,35 @@ mod test {
 
     #[test]
     fn test_parse_account_data() {
+        let account_pubkey = Pubkey::new_rand();
         let other_program = Pubkey::new_rand();
         let data = vec![0; 4];
-        assert!(parse_account_data(&other_program, &data).is_err());
+        assert!(parse_account_data(&account_pubkey, &other_program, &data, None).is_err());
 
         let vote_state = VoteState::default();
         let mut vote_account_data: Vec<u8> = vec![0; VoteState::size_of()];
         let versioned = VoteStateVersions::Current(Box::new(vote_state));
         VoteState::serialize(&versioned, &mut vote_account_data).unwrap();
-        let parsed = parse_account_data(&solana_vote_program::id(), &vote_account_data).unwrap();
+        let parsed = parse_account_data(
+            &account_pubkey,
+            &solana_vote_program::id(),
+            &vote_account_data,
+            None,
+        )
+        .unwrap();
         assert_eq!(parsed.program, "vote".to_string());
+        assert_eq!(parsed.space, VoteState::size_of() as u64);
 
         let nonce_data = Versions::new_current(State::Initialized(Data::default()));
         let nonce_account_data = bincode::serialize(&nonce_data).unwrap();
-        let parsed = parse_account_data(&system_program::id(), &nonce_account_data).unwrap();
+        let parsed = parse_account_data(
+            &account_pubkey,
+            &system_program::id(),
+            &nonce_account_data,
+            None,
+        )
+        .unwrap();
         assert_eq!(parsed.program, "nonce".to_string());
+        assert_eq!(parsed.space, State::size() as u64);
     }
 }

account-decoder/src/parse_config.rs

@@ -0,0 +1,146 @@
+use crate::{
+    parse_account_data::{ParsableAccount, ParseAccountError},
+    validator_info,
+};
+use bincode::deserialize;
+use serde_json::Value;
+use solana_config_program::{get_config_data, ConfigKeys};
+use solana_sdk::pubkey::Pubkey;
+use solana_stake_program::config::Config as StakeConfig;
+
+pub fn parse_config(data: &[u8], pubkey: &Pubkey) -> Result<ConfigAccountType, ParseAccountError> {
+    let parsed_account = if pubkey == &solana_stake_program::config::id() {
+        get_config_data(data)
+            .ok()
+            .and_then(|data| deserialize::<StakeConfig>(data).ok())
+            .map(|config| ConfigAccountType::StakeConfig(config.into()))
+    } else {
+        deserialize::<ConfigKeys>(data).ok().and_then(|key_list| {
+            if !key_list.keys.is_empty() && key_list.keys[0].0 == validator_info::id() {
+                parse_config_data::<String>(data, key_list.keys).and_then(|validator_info| {
+                    Some(ConfigAccountType::ValidatorInfo(UiConfig {
+                        keys: validator_info.keys,
+                        config_data: serde_json::from_str(&validator_info.config_data).ok()?,
+                    }))
+                })
+            } else {
+                None
+            }
+        })
+    };
+    parsed_account.ok_or(ParseAccountError::AccountNotParsable(
+        ParsableAccount::Config,
+    ))
+}
+
+fn parse_config_data<T>(data: &[u8], keys: Vec<(Pubkey, bool)>) -> Option<UiConfig<T>>
+where
+    T: serde::de::DeserializeOwned,
+{
+    let config_data: T = deserialize(&get_config_data(data).ok()?).ok()?;
+    let keys = keys
+        .iter()
+        .map(|key| UiConfigKey {
+            pubkey: key.0.to_string(),
+            signer: key.1,
+        })
+        .collect();
+    Some(UiConfig { keys, config_data })
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase", tag = "type", content = "info")]
+pub enum ConfigAccountType {
+    StakeConfig(UiStakeConfig),
+    ValidatorInfo(UiConfig<Value>),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiConfigKey {
+    pub pubkey: String,
+    pub signer: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiStakeConfig {
+    pub warmup_cooldown_rate: f64,
+    pub slash_penalty: u8,
+}
+
+impl From<StakeConfig> for UiStakeConfig {
+    fn from(config: StakeConfig) -> Self {
+        Self {
+            warmup_cooldown_rate: config.warmup_cooldown_rate,
+            slash_penalty: config.slash_penalty,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiConfig<T> {
+    pub keys: Vec<UiConfigKey>,
+    pub config_data: T,
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::validator_info::ValidatorInfo;
+    use serde_json::json;
+    use solana_config_program::create_config_account;
+
+    #[test]
+    fn test_parse_config() {
+        let stake_config = StakeConfig {
+            warmup_cooldown_rate: 0.25,
+            slash_penalty: 50,
+        };
+        let stake_config_account = create_config_account(vec![], &stake_config, 10);
+        assert_eq!(
+            parse_config(
+                &stake_config_account.data,
+                &solana_stake_program::config::id()
+            )
+            .unwrap(),
+            ConfigAccountType::StakeConfig(UiStakeConfig {
+                warmup_cooldown_rate: 0.25,
+                slash_penalty: 50,
+            }),
+        );
+
+        let validator_info = ValidatorInfo {
+            info: serde_json::to_string(&json!({
+                "name": "Solana",
+            }))
+            .unwrap(),
+        };
+        let info_pubkey = Pubkey::new_rand();
+        let validator_info_config_account = create_config_account(
+            vec![(validator_info::id(), false), (info_pubkey, true)],
+            &validator_info,
+            10,
+        );
+        assert_eq!(
+            parse_config(&validator_info_config_account.data, &info_pubkey).unwrap(),
+            ConfigAccountType::ValidatorInfo(UiConfig {
+                keys: vec![
+                    UiConfigKey {
+                        pubkey: validator_info::id().to_string(),
+                        signer: false,
+                    },
+                    UiConfigKey {
+                        pubkey: info_pubkey.to_string(),
+                        signer: true,
+                    }
+                ],
+                config_data: serde_json::from_str(r#"{"name":"Solana"}"#).unwrap(),
+            }),
+        );
+
+        let bad_data = vec![0; 4];
+        assert!(parse_config(&bad_data, &info_pubkey).is_err());
+    }
+}

account-decoder/src/parse_nonce.rs

@@ -1,6 +1,5 @@
-use crate::parse_account_data::ParseAccountError;
+use crate::{parse_account_data::ParseAccountError, UiFeeCalculator};
 use solana_sdk::{
-    fee_calculator::FeeCalculator,
     instruction::InstructionError,
     nonce::{state::Versions, State},
 };
@@ -14,7 +13,7 @@ pub fn parse_nonce(data: &[u8]) -> Result<UiNonceState, ParseAccountError> {
         State::Initialized(data) => Ok(UiNonceState::Initialized(UiNonceData {
             authority: data.authority.to_string(),
             blockhash: data.blockhash.to_string(),
-            fee_calculator: data.fee_calculator,
+            fee_calculator: data.fee_calculator.into(),
         })),
     }
 }
@@ -32,7 +31,7 @@ pub enum UiNonceState {
 pub struct UiNonceData {
     pub authority: String,
     pub blockhash: String,
-    pub fee_calculator: FeeCalculator,
+    pub fee_calculator: UiFeeCalculator,
 }
 
 #[cfg(test)]
@@ -56,7 +55,9 @@ mod test {
             UiNonceState::Initialized(UiNonceData {
                 authority: Pubkey::default().to_string(),
                 blockhash: Hash::default().to_string(),
-                fee_calculator: FeeCalculator::default(),
+                fee_calculator: UiFeeCalculator {
+                    lamports_per_signature: 0.to_string(),
+                },
             }),
         );
 

account-decoder/src/parse_stake.rs

@@ -0,0 +1,236 @@
+use crate::{
+    parse_account_data::{ParsableAccount, ParseAccountError},
+    StringAmount,
+};
+use bincode::deserialize;
+use solana_sdk::clock::{Epoch, UnixTimestamp};
+use solana_stake_program::stake_state::{Authorized, Delegation, Lockup, Meta, Stake, StakeState};
+
+pub fn parse_stake(data: &[u8]) -> Result<StakeAccountType, ParseAccountError> {
+    let stake_state: StakeState = deserialize(data)
+        .map_err(|_| ParseAccountError::AccountNotParsable(ParsableAccount::Stake))?;
+    let parsed_account = match stake_state {
+        StakeState::Uninitialized => StakeAccountType::Uninitialized,
+        StakeState::Initialized(meta) => StakeAccountType::Initialized(UiStakeAccount {
+            meta: meta.into(),
+            stake: None,
+        }),
+        StakeState::Stake(meta, stake) => StakeAccountType::Delegated(UiStakeAccount {
+            meta: meta.into(),
+            stake: Some(stake.into()),
+        }),
+        StakeState::RewardsPool => StakeAccountType::RewardsPool,
+    };
+    Ok(parsed_account)
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase", tag = "type", content = "info")]
+#[allow(clippy::large_enum_variant)]
+pub enum StakeAccountType {
+    Uninitialized,
+    Initialized(UiStakeAccount),
+    Delegated(UiStakeAccount),
+    RewardsPool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiStakeAccount {
+    pub meta: UiMeta,
+    pub stake: Option<UiStake>,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiMeta {
+    pub rent_exempt_reserve: StringAmount,
+    pub authorized: UiAuthorized,
+    pub lockup: UiLockup,
+}
+
+impl From<Meta> for UiMeta {
+    fn from(meta: Meta) -> Self {
+        Self {
+            rent_exempt_reserve: meta.rent_exempt_reserve.to_string(),
+            authorized: meta.authorized.into(),
+            lockup: meta.lockup.into(),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiLockup {
+    pub unix_timestamp: UnixTimestamp,
+    pub epoch: Epoch,
+    pub custodian: String,
+}
+
+impl From<Lockup> for UiLockup {
+    fn from(lockup: Lockup) -> Self {
+        Self {
+            unix_timestamp: lockup.unix_timestamp,
+            epoch: lockup.epoch,
+            custodian: lockup.custodian.to_string(),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiAuthorized {
+    pub staker: String,
+    pub withdrawer: String,
+}
+
+impl From<Authorized> for UiAuthorized {
+    fn from(authorized: Authorized) -> Self {
+        Self {
+            staker: authorized.staker.to_string(),
+            withdrawer: authorized.withdrawer.to_string(),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiStake {
+    pub delegation: UiDelegation,
+    pub credits_observed: u64,
+}
+
+impl From<Stake> for UiStake {
+    fn from(stake: Stake) -> Self {
+        Self {
+            delegation: stake.delegation.into(),
+            credits_observed: stake.credits_observed,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiDelegation {
+    pub voter: String,
+    pub stake: StringAmount,
+    pub activation_epoch: StringAmount,
+    pub deactivation_epoch: StringAmount,
+    pub warmup_cooldown_rate: f64,
+}
+
+impl From<Delegation> for UiDelegation {
+    fn from(delegation: Delegation) -> Self {
+        Self {
+            voter: delegation.voter_pubkey.to_string(),
+            stake: delegation.stake.to_string(),
+            activation_epoch: delegation.activation_epoch.to_string(),
+            deactivation_epoch: delegation.deactivation_epoch.to_string(),
+            warmup_cooldown_rate: delegation.warmup_cooldown_rate,
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use bincode::serialize;
+    use solana_sdk::pubkey::Pubkey;
+
+    #[test]
+    fn test_parse_stake() {
+        let stake_state = StakeState::Uninitialized;
+        let stake_data = serialize(&stake_state).unwrap();
+        assert_eq!(
+            parse_stake(&stake_data).unwrap(),
+            StakeAccountType::Uninitialized
+        );
+
+        let pubkey = Pubkey::new_rand();
+        let custodian = Pubkey::new_rand();
+        let authorized = Authorized::auto(&pubkey);
+        let lockup = Lockup {
+            unix_timestamp: 0,
+            epoch: 1,
+            custodian,
+        };
+        let meta = Meta {
+            rent_exempt_reserve: 42,
+            authorized,
+            lockup,
+        };
+
+        let stake_state = StakeState::Initialized(meta);
+        let stake_data = serialize(&stake_state).unwrap();
+        assert_eq!(
+            parse_stake(&stake_data).unwrap(),
+            StakeAccountType::Initialized(UiStakeAccount {
+                meta: UiMeta {
+                    rent_exempt_reserve: 42.to_string(),
+                    authorized: UiAuthorized {
+                        staker: pubkey.to_string(),
+                        withdrawer: pubkey.to_string(),
+                    },
+                    lockup: UiLockup {
+                        unix_timestamp: 0,
+                        epoch: 1,
+                        custodian: custodian.to_string(),
+                    }
+                },
+                stake: None,
+            })
+        );
+
+        let voter_pubkey = Pubkey::new_rand();
+        let stake = Stake {
+            delegation: Delegation {
+                voter_pubkey,
+                stake: 20,
+                activation_epoch: 2,
+                deactivation_epoch: std::u64::MAX,
+                warmup_cooldown_rate: 0.25,
+            },
+            credits_observed: 10,
+        };
+
+        let stake_state = StakeState::Stake(meta, stake);
+        let stake_data = serialize(&stake_state).unwrap();
+        assert_eq!(
+            parse_stake(&stake_data).unwrap(),
+            StakeAccountType::Delegated(UiStakeAccount {
+                meta: UiMeta {
+                    rent_exempt_reserve: 42.to_string(),
+                    authorized: UiAuthorized {
+                        staker: pubkey.to_string(),
+                        withdrawer: pubkey.to_string(),
+                    },
+                    lockup: UiLockup {
+                        unix_timestamp: 0,
+                        epoch: 1,
+                        custodian: custodian.to_string(),
+                    }
+                },
+                stake: Some(UiStake {
+                    delegation: UiDelegation {
+                        voter: voter_pubkey.to_string(),
+                        stake: 20.to_string(),
+                        activation_epoch: 2.to_string(),
+                        deactivation_epoch: std::u64::MAX.to_string(),
+                        warmup_cooldown_rate: 0.25,
+                    },
+                    credits_observed: 10,
+                })
+            })
+        );
+
+        let stake_state = StakeState::RewardsPool;
+        let stake_data = serialize(&stake_state).unwrap();
+        assert_eq!(
+            parse_stake(&stake_data).unwrap(),
+            StakeAccountType::RewardsPool
+        );
+
+        let bad_data = vec![1, 2, 3, 4];
+        assert!(parse_stake(&bad_data).is_err());
+    }
+}

account-decoder/src/parse_sysvar.rs

@@ -0,0 +1,328 @@
+use crate::{
+    parse_account_data::{ParsableAccount, ParseAccountError},
+    StringAmount, UiFeeCalculator,
+};
+use bincode::deserialize;
+use bv::BitVec;
+use solana_sdk::{
+    clock::{Clock, Epoch, Slot, UnixTimestamp},
+    epoch_schedule::EpochSchedule,
+    pubkey::Pubkey,
+    rent::Rent,
+    slot_hashes::SlotHashes,
+    slot_history::{self, SlotHistory},
+    stake_history::{StakeHistory, StakeHistoryEntry},
+    sysvar::{self, fees::Fees, recent_blockhashes::RecentBlockhashes, rewards::Rewards},
+};
+
+pub fn parse_sysvar(data: &[u8], pubkey: &Pubkey) -> Result<SysvarAccountType, ParseAccountError> {
+    let parsed_account = {
+        if pubkey == &sysvar::clock::id() {
+            deserialize::<Clock>(data)
+                .ok()
+                .map(|clock| SysvarAccountType::Clock(clock.into()))
+        } else if pubkey == &sysvar::epoch_schedule::id() {
+            deserialize(data).ok().map(SysvarAccountType::EpochSchedule)
+        } else if pubkey == &sysvar::fees::id() {
+            deserialize::<Fees>(data)
+                .ok()
+                .map(|fees| SysvarAccountType::Fees(fees.into()))
+        } else if pubkey == &sysvar::recent_blockhashes::id() {
+            deserialize::<RecentBlockhashes>(data)
+                .ok()
+                .map(|recent_blockhashes| {
+                    let recent_blockhashes = recent_blockhashes
+                        .iter()
+                        .map(|entry| UiRecentBlockhashesEntry {
+                            blockhash: entry.blockhash.to_string(),
+                            fee_calculator: entry.fee_calculator.clone().into(),
+                        })
+                        .collect();
+                    SysvarAccountType::RecentBlockhashes(recent_blockhashes)
+                })
+        } else if pubkey == &sysvar::rent::id() {
+            deserialize::<Rent>(data)
+                .ok()
+                .map(|rent| SysvarAccountType::Rent(rent.into()))
+        } else if pubkey == &sysvar::rewards::id() {
+            deserialize::<Rewards>(data)
+                .ok()
+                .map(|rewards| SysvarAccountType::Rewards(rewards.into()))
+        } else if pubkey == &sysvar::slot_hashes::id() {
+            deserialize::<SlotHashes>(data).ok().map(|slot_hashes| {
+                let slot_hashes = slot_hashes
+                    .iter()
+                    .map(|slot_hash| UiSlotHashEntry {
+                        slot: slot_hash.0,
+                        hash: slot_hash.1.to_string(),
+                    })
+                    .collect();
+                SysvarAccountType::SlotHashes(slot_hashes)
+            })
+        } else if pubkey == &sysvar::slot_history::id() {
+            deserialize::<SlotHistory>(data).ok().map(|slot_history| {
+                SysvarAccountType::SlotHistory(UiSlotHistory {
+                    next_slot: slot_history.next_slot,
+                    bits: format!("{:?}", SlotHistoryBits(slot_history.bits)),
+                })
+            })
+        } else if pubkey == &sysvar::stake_history::id() {
+            deserialize::<StakeHistory>(data).ok().map(|stake_history| {
+                let stake_history = stake_history
+                    .iter()
+                    .map(|entry| UiStakeHistoryEntry {
+                        epoch: entry.0,
+                        stake_history: entry.1.clone(),
+                    })
+                    .collect();
+                SysvarAccountType::StakeHistory(stake_history)
+            })
+        } else {
+            None
+        }
+    };
+    parsed_account.ok_or(ParseAccountError::AccountNotParsable(
+        ParsableAccount::Sysvar,
+    ))
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase", tag = "type", content = "info")]
+pub enum SysvarAccountType {
+    Clock(UiClock),
+    EpochSchedule(EpochSchedule),
+    Fees(UiFees),
+    RecentBlockhashes(Vec<UiRecentBlockhashesEntry>),
+    Rent(UiRent),
+    Rewards(UiRewards),
+    SlotHashes(Vec<UiSlotHashEntry>),
+    SlotHistory(UiSlotHistory),
+    StakeHistory(Vec<UiStakeHistoryEntry>),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct UiClock {
+    pub slot: Slot,
+    pub epoch: Epoch,
+    pub leader_schedule_epoch: Epoch,
+    pub unix_timestamp: UnixTimestamp,
+}
+
+impl From<Clock> for UiClock {
+    fn from(clock: Clock) -> Self {
+        Self {
+            slot: clock.slot,
+            epoch: clock.epoch,
+            leader_schedule_epoch: clock.leader_schedule_epoch,
+            unix_timestamp: clock.unix_timestamp,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct UiFees {
+    pub fee_calculator: UiFeeCalculator,
+}
+impl From<Fees> for UiFees {
+    fn from(fees: Fees) -> Self {
+        Self {
+            fee_calculator: fees.fee_calculator.into(),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct UiRent {
+    pub lamports_per_byte_year: StringAmount,
+    pub exemption_threshold: f64,
+    pub burn_percent: u8,
+}
+
+impl From<Rent> for UiRent {
+    fn from(rent: Rent) -> Self {
+        Self {
+            lamports_per_byte_year: rent.lamports_per_byte_year.to_string(),
+            exemption_threshold: rent.exemption_threshold,
+            burn_percent: rent.burn_percent,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct UiRewards {
+    pub validator_point_value: f64,
+}
+
+impl From<Rewards> for UiRewards {
+    fn from(rewards: Rewards) -> Self {
+        Self {
+            validator_point_value: rewards.validator_point_value,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiRecentBlockhashesEntry {
+    pub blockhash: String,
+    pub fee_calculator: UiFeeCalculator,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiSlotHashEntry {
+    pub slot: Slot,
+    pub hash: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiSlotHistory {
+    pub next_slot: Slot,
+    pub bits: String,
+}
+
+struct SlotHistoryBits(BitVec<u64>);
+
+impl std::fmt::Debug for SlotHistoryBits {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        for i in 0..slot_history::MAX_ENTRIES {
+            if self.0.get(i) {
+                write!(f, "1")?;
+            } else {
+                write!(f, "0")?;
+            }
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiStakeHistoryEntry {
+    pub epoch: Epoch,
+    pub stake_history: StakeHistoryEntry,
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use solana_sdk::{
+        fee_calculator::FeeCalculator,
+        hash::Hash,
+        sysvar::{recent_blockhashes::IterItem, Sysvar},
+    };
+    use std::iter::FromIterator;
+
+    #[test]
+    fn test_parse_sysvars() {
+        let clock_sysvar = Clock::default().create_account(1);
+        assert_eq!(
+            parse_sysvar(&clock_sysvar.data, &sysvar::clock::id()).unwrap(),
+            SysvarAccountType::Clock(UiClock::default()),
+        );
+
+        let epoch_schedule = EpochSchedule {
+            slots_per_epoch: 12,
+            leader_schedule_slot_offset: 0,
+            warmup: false,
+            first_normal_epoch: 1,
+            first_normal_slot: 12,
+        };
+        let epoch_schedule_sysvar = epoch_schedule.create_account(1);
+        assert_eq!(
+            parse_sysvar(&epoch_schedule_sysvar.data, &sysvar::epoch_schedule::id()).unwrap(),
+            SysvarAccountType::EpochSchedule(epoch_schedule),
+        );
+
+        let fees_sysvar = Fees::default().create_account(1);
+        assert_eq!(
+            parse_sysvar(&fees_sysvar.data, &sysvar::fees::id()).unwrap(),
+            SysvarAccountType::Fees(UiFees::default()),
+        );
+
+        let hash = Hash::new(&[1; 32]);
+        let fee_calculator = FeeCalculator {
+            lamports_per_signature: 10,
+        };
+        let recent_blockhashes =
+            RecentBlockhashes::from_iter(vec![IterItem(0, &hash, &fee_calculator)].into_iter());
+        let recent_blockhashes_sysvar = recent_blockhashes.create_account(1);
+        assert_eq!(
+            parse_sysvar(
+                &recent_blockhashes_sysvar.data,
+                &sysvar::recent_blockhashes::id()
+            )
+            .unwrap(),
+            SysvarAccountType::RecentBlockhashes(vec![UiRecentBlockhashesEntry {
+                blockhash: hash.to_string(),
+                fee_calculator: fee_calculator.into(),
+            }]),
+        );
+
+        let rent = Rent {
+            lamports_per_byte_year: 10,
+            exemption_threshold: 2.0,
+            burn_percent: 5,
+        };
+        let rent_sysvar = rent.create_account(1);
+        assert_eq!(
+            parse_sysvar(&rent_sysvar.data, &sysvar::rent::id()).unwrap(),
+            SysvarAccountType::Rent(rent.into()),
+        );
+
+        let rewards_sysvar = Rewards::default().create_account(1);
+        assert_eq!(
+            parse_sysvar(&rewards_sysvar.data, &sysvar::rewards::id()).unwrap(),
+            SysvarAccountType::Rewards(UiRewards::default()),
+        );
+
+        let mut slot_hashes = SlotHashes::default();
+        slot_hashes.add(1, hash);
+        let slot_hashes_sysvar = slot_hashes.create_account(1);
+        assert_eq!(
+            parse_sysvar(&slot_hashes_sysvar.data, &sysvar::slot_hashes::id()).unwrap(),
+            SysvarAccountType::SlotHashes(vec![UiSlotHashEntry {
+                slot: 1,
+                hash: hash.to_string(),
+            }]),
+        );
+
+        let mut slot_history = SlotHistory::default();
+        slot_history.add(42);
+        let slot_history_sysvar = slot_history.create_account(1);
+        assert_eq!(
+            parse_sysvar(&slot_history_sysvar.data, &sysvar::slot_history::id()).unwrap(),
+            SysvarAccountType::SlotHistory(UiSlotHistory {
+                next_slot: slot_history.next_slot,
+                bits: format!("{:?}", SlotHistoryBits(slot_history.bits)),
+            }),
+        );
+
+        let mut stake_history = StakeHistory::default();
+        let stake_history_entry = StakeHistoryEntry {
+            effective: 10,
+            activating: 2,
+            deactivating: 3,
+        };
+        stake_history.add(1, stake_history_entry.clone());
+        let stake_history_sysvar = stake_history.create_account(1);
+        assert_eq!(
+            parse_sysvar(&stake_history_sysvar.data, &sysvar::stake_history::id()).unwrap(),
+            SysvarAccountType::StakeHistory(vec![UiStakeHistoryEntry {
+                epoch: 1,
+                stake_history: stake_history_entry,
+            }]),
+        );
+
+        let bad_pubkey = Pubkey::new_rand();
+        assert!(parse_sysvar(&stake_history_sysvar.data, &bad_pubkey).is_err());
+
+        let bad_data = vec![0; 4];
+        assert!(parse_sysvar(&bad_data, &sysvar::stake_history::id()).is_err());
+    }
+}

account-decoder/src/parse_token.rs

@@ -1,4 +1,7 @@
-use crate::parse_account_data::{ParsableAccount, ParseAccountError};
+use crate::{
+    parse_account_data::{ParsableAccount, ParseAccountError},
+    StringAmount,
+};
 use solana_sdk::pubkey::Pubkey;
 use spl_token_v1_0::{
     option::COption,
@@ -19,22 +22,37 @@ pub fn spl_token_v1_0_native_mint() -> Pubkey {
     Pubkey::from_str(&spl_token_v1_0::native_mint::id().to_string()).unwrap()
 }
 
-pub fn parse_token(data: &[u8]) -> Result<TokenAccountType, ParseAccountError> {
+pub fn parse_token(
+    data: &[u8],
+    mint_decimals: Option<u8>,
+) -> Result<TokenAccountType, ParseAccountError> {
     let mut data = data.to_vec();
     if data.len() == size_of::<Account>() {
         let account: Account = *unpack(&mut data)
             .map_err(|_| ParseAccountError::AccountNotParsable(ParsableAccount::SplToken))?;
+        let decimals = mint_decimals.ok_or_else(|| {
+            ParseAccountError::AdditionalDataMissing(
+                "no mint_decimals provided to parse spl-token account".to_string(),
+            )
+        })?;
         Ok(TokenAccountType::Account(UiTokenAccount {
             mint: account.mint.to_string(),
             owner: account.owner.to_string(),
-            amount: account.amount,
+            token_amount: token_amount_to_ui_amount(account.amount, decimals),
             delegate: match account.delegate {
                 COption::Some(pubkey) => Some(pubkey.to_string()),
                 COption::None => None,
             },
             is_initialized: account.is_initialized,
             is_native: account.is_native,
-            delegated_amount: account.delegated_amount,
+            delegated_amount: if account.delegate.is_none() {
+                None
+            } else {
+                Some(token_amount_to_ui_amount(
+                    account.delegated_amount,
+                    decimals,
+                ))
+            },
         }))
     } else if data.len() == size_of::<Mint>() {
         let mint: Mint = *unpack(&mut data)
@@ -86,11 +104,31 @@ pub enum TokenAccountType {
 pub struct UiTokenAccount {
     pub mint: String,
     pub owner: String,
-    pub amount: u64,
+    pub token_amount: UiTokenAmount,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub delegate: Option<String>,
     pub is_initialized: bool,
     pub is_native: bool,
-    pub delegated_amount: u64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub delegated_amount: Option<UiTokenAmount>,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiTokenAmount {
+    pub ui_amount: f64,
+    pub decimals: u8,
+    pub amount: StringAmount,
+}
+
+pub fn token_amount_to_ui_amount(amount: u64, decimals: u8) -> UiTokenAmount {
+    // Use `amount_to_ui_amount()` once spl_token is bumped to a version that supports it: https://github.com/solana-labs/solana-program-library/pull/211
+    let amount_decimals = amount as f64 / 10_usize.pow(decimals as u32) as f64;
+    UiTokenAmount {
+        ui_amount: amount_decimals,
+        decimals,
+        amount: amount.to_string(),
+    }
 }
 
 #[derive(Debug, Serialize, Deserialize, PartialEq)]
@@ -110,6 +148,14 @@ pub struct UiMultisig {
     pub signers: Vec<String>,
 }
 
+pub fn get_token_account_mint(data: &[u8]) -> Option<Pubkey> {
+    if data.len() == size_of::<Account>() {
+        Some(Pubkey::new(&data[0..32]))
+    } else {
+        None
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -125,16 +171,21 @@ mod test {
         account.owner = owner_pubkey;
         account.amount = 42;
         account.is_initialized = true;
+        assert!(parse_token(&account_data, None).is_err());
         assert_eq!(
-            parse_token(&account_data).unwrap(),
+            parse_token(&account_data, Some(2)).unwrap(),
             TokenAccountType::Account(UiTokenAccount {
                 mint: mint_pubkey.to_string(),
                 owner: owner_pubkey.to_string(),
-                amount: 42,
+                token_amount: UiTokenAmount {
+                    ui_amount: 0.42,
+                    decimals: 2,
+                    amount: "42".to_string()
+                },
                 delegate: None,
                 is_initialized: true,
                 is_native: false,
-                delegated_amount: 0,
+                delegated_amount: None,
             }),
         );
 
@@ -144,7 +195,7 @@ mod test {
         mint.decimals = 3;
         mint.is_initialized = true;
         assert_eq!(
-            parse_token(&mint_data).unwrap(),
+            parse_token(&mint_data, None).unwrap(),
             TokenAccountType::Mint(UiMint {
                 owner: Some(owner_pubkey.to_string()),
                 decimals: 3,
@@ -166,7 +217,7 @@ mod test {
         multisig.is_initialized = true;
         multisig.signers = signers;
         assert_eq!(
-            parse_token(&multisig_data).unwrap(),
+            parse_token(&multisig_data, None).unwrap(),
             TokenAccountType::Multisig(UiMultisig {
                 num_required_signers: 2,
                 num_valid_signers: 3,
@@ -180,6 +231,20 @@ mod test {
         );
 
         let bad_data = vec![0; 4];
-        assert!(parse_token(&bad_data).is_err());
+        assert!(parse_token(&bad_data, None).is_err());
+    }
+
+    #[test]
+    fn test_get_token_account_mint() {
+        let mint_pubkey = SplTokenPubkey::new(&[2; 32]);
+        let mut account_data = [0; size_of::<Account>()];
+        let mut account: &mut Account = unpack_unchecked(&mut account_data).unwrap();
+        account.mint = mint_pubkey;
+
+        let expected_mint_pubkey = Pubkey::new(&[2; 32]);
+        assert_eq!(
+            get_token_account_mint(&account_data),
+            Some(expected_mint_pubkey)
+        );
     }
 }

account-decoder/src/parse_vote.rs

@@ -1,4 +1,4 @@
-use crate::parse_account_data::ParseAccountError;
+use crate::{parse_account_data::ParseAccountError, StringAmount};
 use solana_sdk::{
     clock::{Epoch, Slot},
     pubkey::Pubkey,
@@ -12,8 +12,8 @@ pub fn parse_vote(data: &[u8]) -> Result<VoteAccountType, ParseAccountError> {
         .iter()
         .map(|(epoch, credits, previous_credits)| UiEpochCredits {
             epoch: *epoch,
-            credits: *credits,
-            previous_credits: *previous_credits,
+            credits: credits.to_string(),
+            previous_credits: previous_credits.to_string(),
         })
         .collect();
     let votes = vote_state
@@ -115,8 +115,8 @@ struct UiPriorVoters {
 #[serde(rename_all = "camelCase")]
 struct UiEpochCredits {
     epoch: Epoch,
-    credits: u64,
-    previous_credits: u64,
+    credits: StringAmount,
+    previous_credits: StringAmount,
 }
 
 #[cfg(test)]

account-decoder/src/validator_info.rs

@@ -0,0 +1,18 @@
+use solana_config_program::ConfigState;
+
+pub const MAX_SHORT_FIELD_LENGTH: usize = 70;
+pub const MAX_LONG_FIELD_LENGTH: usize = 300;
+pub const MAX_VALIDATOR_INFO: u64 = 576;
+
+solana_sdk::declare_id!("Va1idator1nfo111111111111111111111111111111");
+
+#[derive(Debug, Deserialize, PartialEq, Serialize, Default)]
+pub struct ValidatorInfo {
+    pub info: String,
+}
+
+impl ConfigState for ValidatorInfo {
+    fn max_space() -> u64 {
+        MAX_VALIDATOR_INFO
+    }
+}

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,10 +10,10 @@ homepage = "https://solana.com/"
 [dependencies]
 log = "0.4.6"
 rayon = "1.3.0"
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-runtime = { path = "../runtime", version = "1.2.19" }
-solana-measure = { path = "../measure", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-runtime = { path = "../runtime", version = "1.2.24" }
+solana-measure = { path = "../measure", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,16 +13,16 @@ crossbeam-channel = "0.4"
 log = "0.4.6"
 rand = "0.7.0"
 rayon = "1.3.0"
-solana-core = { path = "../core", version = "1.2.19" }
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-streamer = { path = "../streamer", version = "1.2.19" }
-solana-perf = { path = "../perf", version = "1.2.19" }
-solana-ledger = { path = "../ledger", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-runtime = { path = "../runtime", version = "1.2.19" }
-solana-measure = { path = "../measure", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
+solana-core = { path = "../core", version = "1.2.24" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-streamer = { path = "../streamer", version = "1.2.24" }
+solana-perf = { path = "../perf", version = "1.2.24" }
+solana-ledger = { path = "../ledger", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-runtime = { path = "../runtime", version = "1.2.24" }
+solana-measure = { path = "../measure", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,21 +18,21 @@ rand = "0.7.0"
 rayon = "1.3.0"
 serde_json = "1.0.53"
 serde_yaml = "0.8.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-core = { path = "../core", version = "1.2.19" }
-solana-genesis = { path = "../genesis", version = "1.2.19" }
-solana-client = { path = "../client", version = "1.2.19" }
-solana-faucet = { path = "../faucet", version = "1.2.19" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-metrics = { path = "../metrics", version = "1.2.19" }
-solana-net-utils = { path = "../net-utils", version = "1.2.19" }
-solana-runtime = { path = "../runtime", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-core = { path = "../core", version = "1.2.24" }
+solana-genesis = { path = "../genesis", version = "1.2.24" }
+solana-client = { path = "../client", version = "1.2.24" }
+solana-faucet = { path = "../faucet", version = "1.2.24" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-metrics = { path = "../metrics", version = "1.2.24" }
+solana-net-utils = { path = "../net-utils", version = "1.2.24" }
+solana-runtime = { path = "../runtime", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.2.19" }
+solana-local-cluster = { path = "../local-cluster", version = "1.2.24" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/Cargo.toml

@@ -2,18 +2,18 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-streamer = { path = "../streamer", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-net-utils = { path = "../net-utils", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-streamer = { path = "../streamer", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-net-utils = { path = "../net-utils", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,23 +14,23 @@ log = "0.4.8"
 rayon = "1.3.0"
 serde_json = "1.0.53"
 serde_yaml = "0.8.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-core = { path = "../core", version = "1.2.19" }
-solana-genesis = { path = "../genesis", version = "1.2.19" }
-solana-client = { path = "../client", version = "1.2.19" }
-solana-faucet = { path = "../faucet", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-metrics = { path = "../metrics", version = "1.2.19" }
-solana-measure = { path = "../measure", version = "1.2.19" }
-solana-net-utils = { path = "../net-utils", version = "1.2.19" }
-solana-runtime = { path = "../runtime", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-core = { path = "../core", version = "1.2.24" }
+solana-genesis = { path = "../genesis", version = "1.2.24" }
+solana-client = { path = "../client", version = "1.2.24" }
+solana-faucet = { path = "../faucet", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-metrics = { path = "../metrics", version = "1.2.24" }
+solana-measure = { path = "../measure", version = "1.2.24" }
+solana-net-utils = { path = "../net-utils", version = "1.2.24" }
+solana-runtime = { path = "../runtime", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.2.19" }
+solana-local-cluster = { path = "../local-cluster", version = "1.2.24" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

ci/run-sanity.sh

@@ -7,7 +7,7 @@ source multinode-demo/common.sh
 
 rm -rf config/run/init-completed config/ledger config/snapshot-ledger
 
-timeout 15 ./run.sh &
+timeout 120 ./run.sh &
 pid=$!
 
 attempts=20

ci/semver_bash/README.md

@@ -27,5 +27,5 @@ Alternatively, you can source it from within a script:
     local PATCH=0  
     local SPECIAL=""
     
-    semverParseInto "1.2.19" MAJOR MINOR PATCH SPECIAL
+    semverParseInto "1.2.24" MAJOR MINOR PATCH SPECIAL
     semverParseInto "3.2.1" MAJOR MINOR PATCH SPECIAL

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
 thiserror = "1.0.11"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -27,29 +27,29 @@ reqwest = { version = "0.10.4", default-features = false, features = ["blocking"
 serde = "1.0.110"
 serde_derive = "1.0.103"
 serde_json = "1.0.53"
-solana-account-decoder = { path = "../account-decoder", version = "1.2.19" }
-solana-budget-program = { path = "../programs/budget", version = "1.2.19" }
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-cli-config = { path = "../cli-config", version = "1.2.19" }
-solana-client = { path = "../client", version = "1.2.19" }
-solana-config-program = { path = "../programs/config", version = "1.2.19" }
-solana-faucet = { path = "../faucet", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-net-utils = { path = "../net-utils", version = "1.2.19" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.2.19" }
-solana-runtime = { path = "../runtime", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-stake-program = { path = "../programs/stake", version = "1.2.19" }
-solana-transaction-status = { path = "../transaction-status", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.19" }
-solana-vote-signer = { path = "../vote-signer", version = "1.2.19" }
+solana-account-decoder = { path = "../account-decoder", version = "1.2.24" }
+solana-budget-program = { path = "../programs/budget", version = "1.2.24" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-cli-config = { path = "../cli-config", version = "1.2.24" }
+solana-client = { path = "../client", version = "1.2.24" }
+solana-config-program = { path = "../programs/config", version = "1.2.24" }
+solana-faucet = { path = "../faucet", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-net-utils = { path = "../net-utils", version = "1.2.24" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.2.24" }
+solana-runtime = { path = "../runtime", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.24" }
+solana-transaction-status = { path = "../transaction-status", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.24" }
+solana-vote-signer = { path = "../vote-signer", version = "1.2.24" }
 thiserror = "1.0.19"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.2.19" }
-solana-budget-program = { path = "../programs/budget", version = "1.2.19" }
+solana-core = { path = "../core", version = "1.2.24" }
+solana-budget-program = { path = "../programs/budget", version = "1.2.24" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cli.rs

@@ -29,7 +29,7 @@ use solana_client::{
     client_error::{ClientError, ClientErrorKind, Result as ClientResult},
     rpc_client::RpcClient,
     rpc_config::{RpcLargestAccountsFilter, RpcSendTransactionConfig},
-    rpc_response::RpcKeyedAccount,
+    rpc_response::{Response, RpcKeyedAccount},
 };
 #[cfg(not(test))]
 use solana_faucet::faucet::request_airdrop_transaction;
@@ -247,6 +247,7 @@ pub enum CliCommand {
     TransactionHistory {
         address: Pubkey,
         before: Option<Signature>,
+        until: Option<Signature>,
         limit: usize,
     },
     // Nonce commands
@@ -1230,7 +1231,13 @@ fn process_show_account(
     let cli_account = CliAccount {
         keyed_account: RpcKeyedAccount {
             pubkey: account_pubkey.to_string(),
-            account: UiAccount::encode(account, UiAccountEncoding::Binary),
+            account: UiAccount::encode(
+                account_pubkey,
+                account,
+                UiAccountEncoding::Binary64,
+                None,
+                None,
+            ),
         },
         use_lamports_unit,
     };
@@ -1308,23 +1315,16 @@ fn send_and_confirm_transactions_with_spinner<T: Signers>(
             transactions_signatures = transactions_signatures
                 .into_iter()
                 .filter(|(_transaction, signature)| {
-                    if let Some(signature) = signature {
-                        if let Ok(status) = rpc_client.get_signature_status(&signature) {
-                            if rpc_client
-                                .get_num_blocks_since_signature_confirmation(&signature)
-                                .unwrap_or(0)
-                                > 1
-                            {
-                                return false;
-                            } else {
-                                return match status {
-                                    None => true,
-                                    Some(result) => result.is_err(),
-                                };
+                    signature
+                        .and_then(|signature| rpc_client.get_signature_statuses(&[signature]).ok())
+                        .map(|Response { context: _, value }| match &value[0] {
+                            None => true,
+                            Some(transaction_status) => {
+                                !(transaction_status.confirmations.is_none()
+                                    || transaction_status.confirmations.unwrap() > 1)
                             }
-                        }
-                    }
-                    true
+                        })
+                        .unwrap_or(true)
                 })
                 .collect();
 
@@ -1379,7 +1379,7 @@ fn process_deploy(
     create_account_tx.try_sign(&[config.signers[0], &program_id], blockhash)?;
     messages.push(&create_account_tx.message);
     let signers = [config.signers[0], &program_id];
-    let mut write_transactions = vec![];
+    let mut write_messages = vec![];
     for (chunk, i) in program_data.chunks(DATA_CHUNK_SIZE).zip(0..) {
         let instruction = loader_instruction::write(
             &program_id.pubkey(),
@@ -1388,19 +1388,17 @@ fn process_deploy(
             chunk.to_vec(),
         );
         let message = Message::new(&[instruction], Some(&signers[0].pubkey()));
-        let mut tx = Transaction::new_unsigned(message);
-        tx.try_sign(&signers, blockhash)?;
-        write_transactions.push(tx);
+        write_messages.push(message);
     }
-    for transaction in write_transactions.iter() {
-        messages.push(&transaction.message);
+    let mut write_message_refs = vec![];
+    for message in write_messages.iter() {
+        write_message_refs.push(message);
     }
+    messages.append(&mut write_message_refs);
 
     let instruction = loader_instruction::finalize(&program_id.pubkey(), &bpf_loader::id());
-    let message = Message::new(&[instruction], Some(&signers[0].pubkey()));
-    let mut finalize_tx = Transaction::new_unsigned(message);
-    finalize_tx.try_sign(&signers, blockhash)?;
-    messages.push(&finalize_tx.message);
+    let finalize_message = Message::new(&[instruction], Some(&signers[0].pubkey()));
+    messages.push(&finalize_message);
 
     check_account_for_multiple_fees(
         rpc_client,
@@ -1415,11 +1413,24 @@ fn process_deploy(
         CliError::DynamicProgramError("Program account allocation failed".to_string())
     })?;
 
+    let (blockhash, _) = rpc_client.get_recent_blockhash()?;
+
+    let mut write_transactions = vec![];
+    for message in write_messages.into_iter() {
+        let mut tx = Transaction::new_unsigned(message);
+        tx.try_sign(&signers, blockhash)?;
+        write_transactions.push(tx);
+    }
+
     trace!("Writing program data");
     send_and_confirm_transactions_with_spinner(&rpc_client, write_transactions, &signers).map_err(
         |_| CliError::DynamicProgramError("Data writes to program account failed".to_string()),
     )?;
 
+    let (blockhash, _) = rpc_client.get_recent_blockhash()?;
+    let mut finalize_tx = Transaction::new_unsigned(finalize_message);
+    finalize_tx.try_sign(&signers, blockhash)?;
+
     trace!("Finalizing program account");
     rpc_client
         .send_and_confirm_transaction_with_spinner_and_config(
@@ -1838,8 +1849,9 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         CliCommand::TransactionHistory {
             address,
             before,
+            until,
             limit,
-        } => process_transaction_history(&rpc_client, config, address, *before, *limit),
+        } => process_transaction_history(&rpc_client, config, address, *before, *until, *limit),
 
         // Nonce Commands
 

cli/src/cluster_query.rs

@@ -14,7 +14,7 @@ use solana_clap_utils::{
 };
 use solana_client::{
     pubsub_client::{PubsubClient, SlotInfoMessage},
-    rpc_client::RpcClient,
+    rpc_client::{GetConfirmedSignaturesForAddress2Config, RpcClient},
     rpc_config::{RpcLargestAccountsConfig, RpcLargestAccountsFilter},
 };
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
@@ -256,9 +256,8 @@ impl ClusterQuerySubCommands for App<'_, '_> {
         )
         .subcommand(
             SubCommand::with_name("transaction-history")
-                .about("Show historical transactions affecting the given address, \
-                       ordered based on the slot in which they were confirmed in \
-                       from lowest to highest slot")
+                .about("Show historical transactions affecting the given address \
+                        from newest to oldest")
                 .arg(
                     pubkey!(Arg::with_name("address")
                         .index(1)
@@ -458,12 +457,21 @@ pub fn parse_transaction_history(
         ),
         None => None,
     };
+    let until = match matches.value_of("until") {
+        Some(signature) => Some(
+            signature
+                .parse()
+                .map_err(|err| CliError::BadParameter(format!("Invalid signature: {}", err)))?,
+        ),
+        None => None,
+    };
     let limit = value_t_or_exit!(matches, "limit", usize);
 
     Ok(CliCommandInfo {
         command: CliCommand::TransactionHistory {
             address,
             before,
+            until,
             limit,
         },
         signers: vec![],
@@ -1283,12 +1291,16 @@ pub fn process_transaction_history(
     config: &CliConfig,
     address: &Pubkey,
     before: Option<Signature>,
+    until: Option<Signature>,
     limit: usize,
 ) -> ProcessResult {
     let results = rpc_client.get_confirmed_signatures_for_address2_with_config(
         address,
-        before,
-        Some(limit),
+        GetConfirmedSignaturesForAddress2Config {
+            before,
+            until,
+            limit: Some(limit),
+        },
     )?;
 
     let transactions_found = format!("{} transactions found", results.len());

cli/src/offline/blockhash_query.rs

@@ -345,7 +345,13 @@ mod tests {
         )
         .unwrap();
         let nonce_pubkey = Pubkey::new(&[4u8; 32]);
-        let rpc_nonce_account = UiAccount::encode(nonce_account, UiAccountEncoding::Binary);
+        let rpc_nonce_account = UiAccount::encode(
+            &nonce_pubkey,
+            nonce_account,
+            UiAccountEncoding::Binary64,
+            None,
+            None,
+        );
         let get_account_response = json!(Response {
             context: RpcResponseContext { slot: 1 },
             value: json!(Some(rpc_nonce_account)),

cli/src/validator_info.rs

@@ -6,9 +6,10 @@ use crate::{
 use bincode::deserialize;
 use clap::{App, AppSettings, Arg, ArgMatches, SubCommand};
 use reqwest::blocking::Client;
-use serde_derive::{Deserialize, Serialize};
 use serde_json::{Map, Value};
-
+use solana_account_decoder::validator_info::{
+    self, ValidatorInfo, MAX_LONG_FIELD_LENGTH, MAX_SHORT_FIELD_LENGTH,
+};
 use solana_clap_utils::{
     input_parsers::pubkey_of,
     input_validators::{is_pubkey, is_url},
@@ -27,23 +28,6 @@ use solana_sdk::{
 };
 use std::{error, sync::Arc};
 
-pub const MAX_SHORT_FIELD_LENGTH: usize = 70;
-pub const MAX_LONG_FIELD_LENGTH: usize = 300;
-pub const MAX_VALIDATOR_INFO: u64 = 576;
-
-solana_sdk::declare_id!("Va1idator1nfo111111111111111111111111111111");
-
-#[derive(Debug, Deserialize, PartialEq, Serialize, Default)]
-pub struct ValidatorInfo {
-    info: String,
-}
-
-impl ConfigState for ValidatorInfo {
-    fn max_space() -> u64 {
-        MAX_VALIDATOR_INFO
-    }
-}
-
 // Return an error if a validator details are longer than the max length.
 pub fn check_details_length(string: String) -> Result<(), String> {
     if string.len() > MAX_LONG_FIELD_LENGTH {
@@ -289,7 +273,7 @@ pub fn process_set_validator_info(
         .iter()
         .filter(|(_, account)| {
             let key_list: ConfigKeys = deserialize(&account.data).map_err(|_| false).unwrap();
-            key_list.keys.contains(&(id(), false))
+            key_list.keys.contains(&(validator_info::id(), false))
         })
         .find(|(pubkey, account)| {
             let (validator_pubkey, _) = parse_validator_info(&pubkey, &account).unwrap();
@@ -328,7 +312,10 @@ pub fn process_set_validator_info(
     };
 
     let build_message = |lamports| {
-        let keys = vec![(id(), false), (config.signers[0].pubkey(), true)];
+        let keys = vec![
+            (validator_info::id(), false),
+            (config.signers[0].pubkey(), true),
+        ];
         if balance == 0 {
             println!(
                 "Publishing info for Validator {:?}",
@@ -400,7 +387,7 @@ pub fn process_get_validator_info(
                 let key_list: ConfigKeys = deserialize(&validator_info_account.data)
                     .map_err(|_| false)
                     .unwrap();
-                key_list.keys.contains(&(id(), false))
+                key_list.keys.contains(&(validator_info::id(), false))
             })
             .collect()
     };
@@ -502,7 +489,7 @@ mod tests {
     #[test]
     fn test_parse_validator_info() {
         let pubkey = Pubkey::new_rand();
-        let keys = vec![(id(), false), (pubkey, true)];
+        let keys = vec![(validator_info::id(), false), (pubkey, true)];
         let config = ConfigKeys { keys };
 
         let mut info = Map::new();

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,11 +19,11 @@ reqwest = { version = "0.10.4", default-features = false, features = ["blocking"
 serde = "1.0.110"
 serde_derive = "1.0.103"
 serde_json = "1.0.53"
-solana-account-decoder = { path = "../account-decoder", version = "1.2.19" }
-solana-net-utils = { path = "../net-utils", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-transaction-status = { path = "../transaction-status", version = "1.2.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.19" }
+solana-account-decoder = { path = "../account-decoder", version = "1.2.24" }
+solana-net-utils = { path = "../net-utils", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-transaction-status = { path = "../transaction-status", version = "1.2.24" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.24" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -32,7 +32,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "14.1.0"
 jsonrpc-http-server = "14.1.0"
-solana-logger = { path = "../logger", version = "1.2.19" }
+solana-logger = { path = "../logger", version = "1.2.24" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/rpc_client.rs

@@ -2,6 +2,7 @@ use crate::{
     client_error::{ClientError, ClientErrorKind, Result as ClientResult},
     http_sender::HttpSender,
     mock_sender::{MockSender, Mocks},
+    rpc_config::RpcAccountInfoConfig,
     rpc_config::{
         RpcGetConfirmedSignaturesForAddress2Config, RpcLargestAccountsConfig,
         RpcSendTransactionConfig, RpcTokenAccountsFilter,
@@ -15,8 +16,10 @@ use indicatif::{ProgressBar, ProgressStyle};
 use log::*;
 use serde_json::{json, Value};
 use solana_account_decoder::{
-    parse_token::{parse_token, TokenAccountType, UiMint, UiMultisig, UiTokenAccount},
+    parse_token::UiTokenAmount,
     UiAccount,
+    UiAccountData::{Binary, Binary64},
+    UiAccountEncoding,
 };
 use solana_sdk::{
     account::Account,
@@ -293,18 +296,21 @@ impl RpcClient {
         &self,
         address: &Pubkey,
     ) -> ClientResult<Vec<RpcConfirmedTransactionStatusWithSignature>> {
-        self.get_confirmed_signatures_for_address2_with_config(address, None, None)
+        self.get_confirmed_signatures_for_address2_with_config(
+            address,
+            GetConfirmedSignaturesForAddress2Config::default(),
+        )
     }
 
     pub fn get_confirmed_signatures_for_address2_with_config(
         &self,
         address: &Pubkey,
-        before: Option<Signature>,
-        limit: Option<usize>,
+        config: GetConfirmedSignaturesForAddress2Config,
     ) -> ClientResult<Vec<RpcConfirmedTransactionStatusWithSignature>> {
         let config = RpcGetConfirmedSignaturesForAddress2Config {
-            before: before.map(|signature| signature.to_string()),
-            limit,
+            before: config.before.map(|signature| signature.to_string()),
+            until: config.until.map(|signature| signature.to_string()),
+            limit: config.limit,
         };
 
         let result: Vec<RpcConfirmedTransactionStatusWithSignature> = self.send(
@@ -470,9 +476,14 @@ impl RpcClient {
         pubkey: &Pubkey,
         commitment_config: CommitmentConfig,
     ) -> RpcResult<Option<Account>> {
+        let config = RpcAccountInfoConfig {
+            encoding: Some(UiAccountEncoding::Binary64),
+            commitment: Some(commitment_config),
+            data_slice: None,
+        };
         let response = self.sender.send(
             RpcRequest::GetAccountInfo,
-            json!([pubkey.to_string(), commitment_config]),
+            json!([pubkey.to_string(), config]),
         );
 
         response
@@ -484,8 +495,17 @@ impl RpcClient {
                 }
                 let Response {
                     context,
-                    value: rpc_account,
+                    value: mut rpc_account,
                 } = serde_json::from_value::<Response<Option<UiAccount>>>(result_json)?;
+                if let Some(ref mut account) = rpc_account {
+                    if let Binary(_) = &account.data {
+                        let tmp = Binary64(String::new());
+                        match std::mem::replace(&mut account.data, tmp) {
+                            Binary(new_data) => account.data = Binary64(new_data),
+                            _ => panic!("should have gotten binary here."),
+                        }
+                    }
+                }
                 trace!("Response account {:?} {:?}", pubkey, rpc_account);
                 let account = rpc_account.and_then(|rpc_account| rpc_account.decode());
                 Ok(Response {
@@ -690,88 +710,7 @@ impl RpcClient {
         Ok(hash)
     }
 
-    pub fn get_token_account(&self, pubkey: &Pubkey) -> ClientResult<Option<UiTokenAccount>> {
-        Ok(self
-            .get_token_account_with_commitment(pubkey, CommitmentConfig::default())?
-            .value)
-    }
-
-    pub fn get_token_account_with_commitment(
-        &self,
-        pubkey: &Pubkey,
-        commitment_config: CommitmentConfig,
-    ) -> RpcResult<Option<UiTokenAccount>> {
-        let Response {
-            context,
-            value: account,
-        } = self.get_account_with_commitment(pubkey, commitment_config)?;
-
-        Ok(Response {
-            context,
-            value: account
-                .map(|account| match parse_token(&account.data) {
-                    Ok(TokenAccountType::Account(ui_token_account)) => Some(ui_token_account),
-                    _ => None,
-                })
-                .flatten(),
-        })
-    }
-
-    pub fn get_token_mint(&self, pubkey: &Pubkey) -> ClientResult<Option<UiMint>> {
-        Ok(self
-            .get_token_mint_with_commitment(pubkey, CommitmentConfig::default())?
-            .value)
-    }
-
-    pub fn get_token_mint_with_commitment(
-        &self,
-        pubkey: &Pubkey,
-        commitment_config: CommitmentConfig,
-    ) -> RpcResult<Option<UiMint>> {
-        let Response {
-            context,
-            value: account,
-        } = self.get_account_with_commitment(pubkey, commitment_config)?;
-
-        Ok(Response {
-            context,
-            value: account
-                .map(|account| match parse_token(&account.data) {
-                    Ok(TokenAccountType::Mint(ui_token_mint)) => Some(ui_token_mint),
-                    _ => None,
-                })
-                .flatten(),
-        })
-    }
-
-    pub fn get_token_multisig(&self, pubkey: &Pubkey) -> ClientResult<Option<UiMultisig>> {
-        Ok(self
-            .get_token_multisig_with_commitment(pubkey, CommitmentConfig::default())?
-            .value)
-    }
-
-    pub fn get_token_multisig_with_commitment(
-        &self,
-        pubkey: &Pubkey,
-        commitment_config: CommitmentConfig,
-    ) -> RpcResult<Option<UiMultisig>> {
-        let Response {
-            context,
-            value: account,
-        } = self.get_account_with_commitment(pubkey, commitment_config)?;
-
-        Ok(Response {
-            context,
-            value: account
-                .map(|account| match parse_token(&account.data) {
-                    Ok(TokenAccountType::Multisig(ui_token_multisig)) => Some(ui_token_multisig),
-                    _ => None,
-                })
-                .flatten(),
-        })
-    }
-
-    pub fn get_token_account_balance(&self, pubkey: &Pubkey) -> ClientResult<RpcTokenAmount> {
+    pub fn get_token_account_balance(&self, pubkey: &Pubkey) -> ClientResult<UiTokenAmount> {
         Ok(self
             .get_token_account_balance_with_commitment(pubkey, CommitmentConfig::default())?
             .value)
@@ -781,7 +720,7 @@ impl RpcClient {
         &self,
         pubkey: &Pubkey,
         commitment_config: CommitmentConfig,
-    ) -> RpcResult<RpcTokenAmount> {
+    ) -> RpcResult<UiTokenAmount> {
         self.send(
             RpcRequest::GetTokenAccountBalance,
             json!([pubkey.to_string(), commitment_config]),
@@ -792,7 +731,7 @@ impl RpcClient {
         &self,
         delegate: &Pubkey,
         token_account_filter: TokenAccountsFilter,
-    ) -> ClientResult<Vec<(Pubkey, UiTokenAccount)>> {
+    ) -> ClientResult<Vec<RpcKeyedAccount>> {
         Ok(self
             .get_token_accounts_by_delegate_with_commitment(
                 delegate,
@@ -807,39 +746,31 @@ impl RpcClient {
         delegate: &Pubkey,
         token_account_filter: TokenAccountsFilter,
         commitment_config: CommitmentConfig,
-    ) -> RpcResult<Vec<(Pubkey, UiTokenAccount)>> {
+    ) -> RpcResult<Vec<RpcKeyedAccount>> {
         let token_account_filter = match token_account_filter {
             TokenAccountsFilter::Mint(mint) => RpcTokenAccountsFilter::Mint(mint.to_string()),
             TokenAccountsFilter::ProgramId(program_id) => {
                 RpcTokenAccountsFilter::ProgramId(program_id.to_string())
             }
         };
-        let Response {
-            context,
-            value: accounts,
-        } = self.send(
-            RpcRequest::GetTokenAccountsByDelegate,
-            json!([
-                delegate.to_string(),
-                token_account_filter,
-                commitment_config
-            ]),
-        )?;
-        let pubkey_accounts = accounts_to_token_accounts(parse_keyed_accounts(
-            accounts,
-            RpcRequest::GetTokenAccountsByDelegate,
-        )?);
-        Ok(Response {
-            context,
-            value: pubkey_accounts,
-        })
+
+        let config = RpcAccountInfoConfig {
+            encoding: Some(UiAccountEncoding::JsonParsed),
+            commitment: Some(commitment_config),
+            data_slice: None,
+        };
+
+        self.send(
+            RpcRequest::GetTokenAccountsByOwner,
+            json!([delegate.to_string(), token_account_filter, config]),
+        )
     }
 
     pub fn get_token_accounts_by_owner(
         &self,
         owner: &Pubkey,
         token_account_filter: TokenAccountsFilter,
-    ) -> ClientResult<Vec<(Pubkey, UiTokenAccount)>> {
+    ) -> ClientResult<Vec<RpcKeyedAccount>> {
         Ok(self
             .get_token_accounts_by_owner_with_commitment(
                 owner,
@@ -854,31 +785,27 @@ impl RpcClient {
         owner: &Pubkey,
         token_account_filter: TokenAccountsFilter,
         commitment_config: CommitmentConfig,
-    ) -> RpcResult<Vec<(Pubkey, UiTokenAccount)>> {
+    ) -> RpcResult<Vec<RpcKeyedAccount>> {
         let token_account_filter = match token_account_filter {
             TokenAccountsFilter::Mint(mint) => RpcTokenAccountsFilter::Mint(mint.to_string()),
             TokenAccountsFilter::ProgramId(program_id) => {
                 RpcTokenAccountsFilter::ProgramId(program_id.to_string())
             }
         };
-        let Response {
-            context,
-            value: accounts,
-        } = self.send(
+
+        let config = RpcAccountInfoConfig {
+            encoding: Some(UiAccountEncoding::JsonParsed),
+            commitment: Some(commitment_config),
+            data_slice: None,
+        };
+
+        self.send(
             RpcRequest::GetTokenAccountsByOwner,
-            json!([owner.to_string(), token_account_filter, commitment_config]),
-        )?;
-        let pubkey_accounts = accounts_to_token_accounts(parse_keyed_accounts(
-            accounts,
-            RpcRequest::GetTokenAccountsByDelegate,
-        )?);
-        Ok(Response {
-            context,
-            value: pubkey_accounts,
-        })
+            json!([owner.to_string(), token_account_filter, config]),
+        )
     }
 
-    pub fn get_token_supply(&self, mint: &Pubkey) -> ClientResult<RpcTokenAmount> {
+    pub fn get_token_supply(&self, mint: &Pubkey) -> ClientResult<UiTokenAmount> {
         Ok(self
             .get_token_supply_with_commitment(mint, CommitmentConfig::default())?
             .value)
@@ -888,7 +815,7 @@ impl RpcClient {
         &self,
         mint: &Pubkey,
         commitment_config: CommitmentConfig,
-    ) -> RpcResult<RpcTokenAmount> {
+    ) -> RpcResult<UiTokenAmount> {
         self.send(
             RpcRequest::GetTokenSupply,
             json!([mint.to_string(), commitment_config]),
@@ -1220,6 +1147,13 @@ impl RpcClient {
     }
 }
 
+#[derive(Debug, Default)]
+pub struct GetConfirmedSignaturesForAddress2Config {
+    pub before: Option<Signature>,
+    pub until: Option<Signature>,
+    pub limit: Option<usize>,
+}
+
 fn new_spinner_progress_bar() -> ProgressBar {
     let progress_bar = ProgressBar::new(42);
     progress_bar
@@ -1261,18 +1195,6 @@ fn parse_keyed_accounts(
     Ok(pubkey_accounts)
 }
 
-fn accounts_to_token_accounts(
-    pubkey_accounts: Vec<(Pubkey, Account)>,
-) -> Vec<(Pubkey, UiTokenAccount)> {
-    pubkey_accounts
-        .into_iter()
-        .filter_map(|(pubkey, account)| match parse_token(&account.data) {
-            Ok(TokenAccountType::Account(ui_token_account)) => Some((pubkey, ui_token_account)),
-            _ => None,
-        })
-        .collect()
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

client/src/rpc_config.rs

@@ -1,5 +1,5 @@
 use crate::rpc_filter::RpcFilterType;
-use solana_account_decoder::UiAccountEncoding;
+use solana_account_decoder::{UiAccountEncoding, UiDataSliceConfig};
 use solana_sdk::{clock::Epoch, commitment_config::CommitmentConfig};
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -47,6 +47,7 @@ pub struct RpcStakeConfig {
 #[serde(rename_all = "camelCase")]
 pub struct RpcAccountInfoConfig {
     pub encoding: Option<UiAccountEncoding>,
+    pub data_slice: Option<UiDataSliceConfig>,
     #[serde(flatten)]
     pub commitment: Option<CommitmentConfig>,
 }
@@ -70,5 +71,6 @@ pub enum RpcTokenAccountsFilter {
 #[serde(rename_all = "camelCase")]
 pub struct RpcGetConfirmedSignaturesForAddress2Config {
     pub before: Option<String>, // Signature as base-58 string
+    pub until: Option<String>,  // Signature as base-58 string
     pub limit: Option<usize>,
 }

client/src/rpc_response.rs

@@ -1,5 +1,5 @@
 use crate::client_error;
-use solana_account_decoder::UiAccount;
+use solana_account_decoder::{parse_token::UiTokenAmount, UiAccount};
 use solana_sdk::{
     clock::{Epoch, Slot},
     fee_calculator::{FeeCalculator, FeeRateGovernor},
@@ -10,7 +10,6 @@ use solana_transaction_status::ConfirmedTransactionStatusWithSignature;
 use std::{collections::HashMap, net::SocketAddr};
 
 pub type RpcResult<T> = client_error::Result<Response<T>>;
-pub type RpcAmount = String;
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct RpcResponseContext {
@@ -222,20 +221,12 @@ pub struct RpcStakeActivation {
     pub inactive: u64,
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcTokenAmount {
-    pub ui_amount: f64,
-    pub decimals: u8,
-    pub amount: RpcAmount,
-}
-
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcTokenAccountBalance {
     pub address: String,
     #[serde(flatten)]
-    pub amount: RpcTokenAmount,
+    pub amount: UiTokenAmount,
 }
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.19"
+version = "1.2.24"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -42,40 +42,42 @@ regex = "1.3.7"
 serde = "1.0.110"
 serde_derive = "1.0.103"
 serde_json = "1.0.53"
-solana-account-decoder = { path = "../account-decoder", version = "1.2.19" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.2.19" }
-solana-budget-program = { path = "../programs/budget", version = "1.2.19" }
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-client = { path = "../client", version = "1.2.19" }
-solana-faucet = { path = "../faucet", version = "1.2.19" }
-solana-genesis-programs = { path = "../genesis-programs", version = "1.2.19" }
-solana-ledger = { path = "../ledger", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.2.19" }
-solana-metrics = { path = "../metrics", version = "1.2.19" }
-solana-measure = { path = "../measure", version = "1.2.19" }
-solana-net-utils = { path = "../net-utils", version = "1.2.19" }
-solana-perf = { path = "../perf", version = "1.2.19" }
-solana-runtime = { path = "../runtime", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-stake-program = { path = "../programs/stake", version = "1.2.19" }
-solana-streamer = { path = "../streamer", version = "1.2.19" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.2.19" }
-solana-transaction-status = { path = "../transaction-status", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.19" }
-solana-vote-signer = { path = "../vote-signer", version = "1.2.19" }
+solana-account-decoder = { path = "../account-decoder", version = "1.2.24" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.2.24" }
+solana-budget-program = { path = "../programs/budget", version = "1.2.24" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-client = { path = "../client", version = "1.2.24" }
+solana-faucet = { path = "../faucet", version = "1.2.24" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.2.24" }
+solana-ledger = { path = "../ledger", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.2.24" }
+solana-metrics = { path = "../metrics", version = "1.2.24" }
+solana-measure = { path = "../measure", version = "1.2.24" }
+solana-net-utils = { path = "../net-utils", version = "1.2.24" }
+solana-perf = { path = "../perf", version = "1.2.24" }
+solana-runtime = { path = "../runtime", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.24" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "1.2.24" }
+solana-streamer = { path = "../streamer", version = "1.2.24" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.2.24" }
+solana-transaction-status = { path = "../transaction-status", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.24" }
+solana-vote-signer = { path = "../vote-signer", version = "1.2.24" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.24" }
 spl-token-v1-0 = { package = "spl-token", version = "1.0.6", features = ["skip-no-mangle"] }
 tempfile = "3.1.0"
 thiserror = "1.0"
-tokio = "0.1"
-tokio-codec = "0.1"
-tokio-fs = "0.1"
-tokio-io = "0.1"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.19" }
+tokio = { version = "0.2.22", features = ["full"] }
+tokio_01 = { version = "0.1", package = "tokio" }
+tokio_fs_01 = { version = "0.1", package = "tokio-fs" }
+tokio_io_01 = { version = "0.1", package = "tokio-io" }
 trees = "0.2.1"
 
 [dev-dependencies]
+base64 = "0.12.3"
 matches = "0.1.6"
 reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 serial_test = "0.4.0"

core/src/cluster_info.rs

@@ -1490,12 +1490,6 @@ impl ClusterInfo {
             .time_gossip_write_lock("purge", &self.stats.purge)
             .purge(timestamp(), &timeouts);
         inc_new_counter_info!("cluster_info-purge-count", num_purged);
-        let table_size = self.gossip.read().unwrap().crds.table.len();
-        datapoint_debug!(
-            "cluster_info-purge",
-            ("table_size", table_size as i64, i64),
-            ("purge_stake_timeout", timeout as i64, i64)
-        );
     }
 
     /// randomly pick a node and ask them for updates asynchronously
@@ -1743,7 +1737,7 @@ impl ClusterInfo {
                 "generate_pull_responses",
                 &self.stats.generate_pull_responses,
             )
-            .generate_pull_responses(&caller_and_filters);
+            .generate_pull_responses(&caller_and_filters, now);
 
         self.time_gossip_write_lock("process_pull_reqs", &self.stats.process_pull_requests)
             .process_pull_requests(caller_and_filters, now);
@@ -2085,6 +2079,10 @@ impl ClusterInfo {
 
     fn print_reset_stats(&self, last_print: &mut Instant) {
         if last_print.elapsed().as_millis() > 2000 {
+            let (table_size, purged_values_size) = {
+                let r_gossip = self.gossip.read().unwrap();
+                (r_gossip.crds.table.len(), r_gossip.pull.purged_values.len())
+            };
             datapoint_info!(
                 "cluster_info_stats",
                 ("entrypoint", self.stats.entrypoint.clear(), i64),
@@ -2108,6 +2106,8 @@ impl ClusterInfo {
                     self.stats.new_push_requests_num.clear(),
                     i64
                 ),
+                ("table_size", table_size as i64, i64),
+                ("purged_values_size", purged_values_size as i64, i64),
             );
             datapoint_info!(
                 "cluster_info_stats2",

core/src/cluster_slots_service.rs

@@ -0,0 +1,193 @@
+use crate::{cluster_info::ClusterInfo, cluster_slots::ClusterSlots};
+use solana_ledger::{
+    bank_forks::BankForks,
+    blockstore::{Blockstore, CompletedSlotsReceiver},
+};
+use solana_measure::measure::Measure;
+use solana_sdk::{clock::Slot, pubkey::Pubkey};
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        {Arc, RwLock},
+    },
+    thread::sleep,
+    thread::{self, Builder, JoinHandle},
+    time::{Duration, Instant},
+};
+
+#[derive(Default, Debug)]
+struct ClusterSlotsServiceTiming {
+    pub lowest_slot_elapsed: u64,
+    pub update_completed_slots_elapsed: u64,
+}
+
+impl ClusterSlotsServiceTiming {
+    fn update(&mut self, lowest_slot_elapsed: u64, update_completed_slots_elapsed: u64) {
+        self.lowest_slot_elapsed += lowest_slot_elapsed;
+        self.update_completed_slots_elapsed += update_completed_slots_elapsed;
+    }
+}
+
+pub struct ClusterSlotsService {
+    t_cluster_slots_service: JoinHandle<()>,
+}
+
+impl ClusterSlotsService {
+    pub fn new(
+        blockstore: Arc<Blockstore>,
+        cluster_slots: Arc<ClusterSlots>,
+        bank_forks: Arc<RwLock<BankForks>>,
+        cluster_info: Arc<ClusterInfo>,
+        completed_slots_receiver: CompletedSlotsReceiver,
+        exit: Arc<AtomicBool>,
+    ) -> Self {
+        let id = cluster_info.id();
+        Self::initialize_lowest_slot(id, &blockstore, &cluster_info);
+        Self::initialize_epoch_slots(&blockstore, &cluster_info, &completed_slots_receiver);
+        let t_cluster_slots_service = Builder::new()
+            .name("solana-cluster-slots-service".to_string())
+            .spawn(move || {
+                Self::run(
+                    blockstore,
+                    cluster_slots,
+                    bank_forks,
+                    cluster_info,
+                    completed_slots_receiver,
+                    exit,
+                )
+            })
+            .unwrap();
+
+        ClusterSlotsService {
+            t_cluster_slots_service,
+        }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.t_cluster_slots_service.join()
+    }
+
+    fn run(
+        blockstore: Arc<Blockstore>,
+        cluster_slots: Arc<ClusterSlots>,
+        bank_forks: Arc<RwLock<BankForks>>,
+        cluster_info: Arc<ClusterInfo>,
+        completed_slots_receiver: CompletedSlotsReceiver,
+        exit: Arc<AtomicBool>,
+    ) {
+        let mut cluster_slots_service_timing = ClusterSlotsServiceTiming::default();
+        let mut last_stats = Instant::now();
+        loop {
+            if exit.load(Ordering::Relaxed) {
+                break;
+            }
+            let new_root = bank_forks.read().unwrap().root();
+            let id = cluster_info.id();
+            let mut lowest_slot_elapsed = Measure::start("lowest_slot_elapsed");
+            let lowest_slot = blockstore.lowest_slot();
+            Self::update_lowest_slot(&id, lowest_slot, &cluster_info);
+            lowest_slot_elapsed.stop();
+            let mut update_completed_slots_elapsed =
+                Measure::start("update_completed_slots_elapsed");
+            Self::update_completed_slots(&completed_slots_receiver, &cluster_info);
+            cluster_slots.update(new_root, &cluster_info, &bank_forks);
+            update_completed_slots_elapsed.stop();
+
+            cluster_slots_service_timing.update(
+                lowest_slot_elapsed.as_us(),
+                update_completed_slots_elapsed.as_us(),
+            );
+
+            if last_stats.elapsed().as_secs() > 2 {
+                datapoint_info!(
+                    "cluster_slots_service-timing",
+                    (
+                        "lowest_slot_elapsed",
+                        cluster_slots_service_timing.lowest_slot_elapsed,
+                        i64
+                    ),
+                    (
+                        "update_completed_slots_elapsed",
+                        cluster_slots_service_timing.update_completed_slots_elapsed,
+                        i64
+                    ),
+                );
+                cluster_slots_service_timing = ClusterSlotsServiceTiming::default();
+                last_stats = Instant::now();
+            }
+            sleep(Duration::from_millis(200));
+        }
+    }
+
+    fn update_completed_slots(
+        completed_slots_receiver: &CompletedSlotsReceiver,
+        cluster_info: &ClusterInfo,
+    ) {
+        let mut slots: Vec<Slot> = vec![];
+        while let Ok(mut more) = completed_slots_receiver.try_recv() {
+            slots.append(&mut more);
+        }
+        slots.sort();
+        if !slots.is_empty() {
+            cluster_info.push_epoch_slots(&slots);
+        }
+    }
+
+    fn initialize_lowest_slot(id: Pubkey, blockstore: &Blockstore, cluster_info: &ClusterInfo) {
+        // Safe to set into gossip because by this time, the leader schedule cache should
+        // also be updated with the latest root (done in blockstore_processor) and thus
+        // will provide a schedule to window_service for any incoming shreds up to the
+        // last_confirmed_epoch.
+        cluster_info.push_lowest_slot(id, blockstore.lowest_slot());
+    }
+
+    fn update_lowest_slot(id: &Pubkey, lowest_slot: Slot, cluster_info: &ClusterInfo) {
+        cluster_info.push_lowest_slot(*id, lowest_slot);
+    }
+
+    fn initialize_epoch_slots(
+        blockstore: &Blockstore,
+        cluster_info: &ClusterInfo,
+        completed_slots_receiver: &CompletedSlotsReceiver,
+    ) {
+        let root = blockstore.last_root();
+        let mut slots: Vec<_> = blockstore
+            .live_slots_iterator(root)
+            .filter_map(|(slot, slot_meta)| {
+                if slot_meta.is_full() {
+                    Some(slot)
+                } else {
+                    None
+                }
+            })
+            .collect();
+
+        while let Ok(mut more) = completed_slots_receiver.try_recv() {
+            slots.append(&mut more);
+        }
+        slots.sort();
+        slots.dedup();
+        if !slots.is_empty() {
+            cluster_info.push_epoch_slots(&slots);
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::cluster_info::Node;
+
+    #[test]
+    pub fn test_update_lowest_slot() {
+        let node_info = Node::new_localhost_with_pubkey(&Pubkey::default());
+        let cluster_info = ClusterInfo::new_with_invalid_keypair(node_info.info);
+        ClusterSlotsService::update_lowest_slot(&Pubkey::default(), 5, &cluster_info);
+        let lowest = cluster_info
+            .get_lowest_slot_for_node(&Pubkey::default(), None, |lowest_slot, _| {
+                lowest_slot.clone()
+            })
+            .unwrap();
+        assert_eq!(lowest.lowest, 5);
+    }
+}

core/src/commitment_service.rs

@@ -8,6 +8,7 @@ use solana_runtime::bank::Bank;
 use solana_sdk::clock::Slot;
 use solana_vote_program::vote_state::VoteState;
 use std::{
+    cmp::max,
     collections::HashMap,
     sync::atomic::{AtomicBool, Ordering},
     sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender},
@@ -103,27 +104,8 @@ impl AggregateCommitmentService {
             }
 
             let mut aggregate_commitment_time = Measure::start("aggregate-commitment-ms");
-            let (block_commitment, rooted_stake) =
-                Self::aggregate_commitment(&ancestors, &aggregation_data.bank);
-
-            let largest_confirmed_root =
-                get_largest_confirmed_root(rooted_stake, aggregation_data.total_staked);
-
-            let mut new_block_commitment = BlockCommitmentCache::new(
-                block_commitment,
-                largest_confirmed_root,
-                aggregation_data.total_staked,
-                aggregation_data.bank,
-                block_commitment_cache.read().unwrap().blockstore.clone(),
-                aggregation_data.root,
-                aggregation_data.root,
-            );
-            new_block_commitment.highest_confirmed_slot =
-                new_block_commitment.calculate_highest_confirmed_slot();
-
-            let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
-
-            std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
+            let cache_slot_info =
+                Self::update_commitment_cache(block_commitment_cache, aggregation_data, ancestors);
             aggregate_commitment_time.stop();
             datapoint_info!(
                 "block-commitment-cache",
@@ -134,12 +116,50 @@ impl AggregateCommitmentService {
                 )
             );
 
-            subscriptions.notify_subscribers(CacheSlotInfo {
-                current_slot: w_block_commitment_cache.slot(),
-                node_root: w_block_commitment_cache.root(),
-                largest_confirmed_root: w_block_commitment_cache.largest_confirmed_root(),
-                highest_confirmed_slot: w_block_commitment_cache.highest_confirmed_slot(),
-            });
+            // Triggers rpc_subscription notifications as soon as new commitment data is available,
+            // sending just the commitment cache slot information that the notifications thread
+            // needs
+            subscriptions.notify_subscribers(cache_slot_info);
+        }
+    }
+
+    fn update_commitment_cache(
+        block_commitment_cache: &RwLock<BlockCommitmentCache>,
+        aggregation_data: CommitmentAggregationData,
+        ancestors: Vec<u64>,
+    ) -> CacheSlotInfo {
+        let (block_commitment, rooted_stake) =
+            Self::aggregate_commitment(&ancestors, &aggregation_data.bank);
+
+        let largest_confirmed_root =
+            get_largest_confirmed_root(rooted_stake, aggregation_data.total_staked);
+
+        let mut new_block_commitment = BlockCommitmentCache::new(
+            block_commitment,
+            largest_confirmed_root,
+            aggregation_data.total_staked,
+            aggregation_data.bank,
+            block_commitment_cache.read().unwrap().blockstore.clone(),
+            aggregation_data.root,
+            aggregation_data.root,
+        );
+        new_block_commitment.highest_confirmed_slot =
+            new_block_commitment.calculate_highest_confirmed_slot();
+
+        let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
+
+        let largest_confirmed_root = max(
+            new_block_commitment.largest_confirmed_root(),
+            w_block_commitment_cache.largest_confirmed_root(),
+        );
+        new_block_commitment.set_largest_confirmed_root(largest_confirmed_root);
+
+        std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
+        CacheSlotInfo {
+            current_slot: w_block_commitment_cache.slot(),
+            node_root: w_block_commitment_cache.root(),
+            largest_confirmed_root: w_block_commitment_cache.largest_confirmed_root(),
+            highest_confirmed_slot: w_block_commitment_cache.highest_confirmed_slot(),
         }
     }
 
@@ -225,10 +245,24 @@ impl AggregateCommitmentService {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-    use solana_sdk::pubkey::Pubkey;
+    use solana_ledger::{
+        bank_forks::BankForks,
+        blockstore::Blockstore,
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        get_tmp_ledger_path,
+    };
+    use solana_runtime::genesis_utils::{
+        create_genesis_config_with_vote_accounts, ValidatorVoteKeypairs,
+    };
+    use solana_sdk::{
+        pubkey::Pubkey,
+        signature::{Keypair, Signer},
+    };
     use solana_stake_program::stake_state;
-    use solana_vote_program::vote_state::{self, VoteStateVersions};
+    use solana_vote_program::{
+        vote_state::{self, VoteStateVersions},
+        vote_transaction,
+    };
 
     #[test]
     fn test_get_largest_confirmed_root() {
@@ -451,4 +485,163 @@ mod tests {
         assert_eq!(rooted_stake.len(), 2);
         assert_eq!(get_largest_confirmed_root(rooted_stake, 100), 1)
     }
+
+    #[test]
+    fn test_highest_confirmed_root_advance() {
+        fn get_vote_account_root_slot(vote_pubkey: Pubkey, bank: &Arc<Bank>) -> Slot {
+            let account = &bank.vote_accounts()[&vote_pubkey].1;
+            let vote_state = VoteState::from(account).unwrap();
+            vote_state.root_slot.unwrap()
+        }
+
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+            let block_commitment_cache = RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            );
+
+            let node_keypair = Keypair::new().to_bytes();
+            let vote_keypair = Keypair::new().to_bytes();
+            let stake_keypair = Keypair::new().to_bytes();
+            let validator_keypairs = vec![ValidatorVoteKeypairs {
+                node_keypair: Keypair::from_bytes(&node_keypair).unwrap(),
+                vote_keypair: Keypair::from_bytes(&vote_keypair).unwrap(),
+                stake_keypair: Keypair::from_bytes(&stake_keypair).unwrap(),
+            }];
+            let GenesisConfigInfo {
+                genesis_config,
+                mint_keypair,
+                voting_keypair: _,
+            } = create_genesis_config_with_vote_accounts(1_000_000_000, &validator_keypairs, 100);
+
+            let node_keypair = Keypair::from_bytes(&node_keypair).unwrap();
+            let vote_keypair = Keypair::from_bytes(&vote_keypair).unwrap();
+
+            let bank0 = Bank::new(&genesis_config);
+            bank0
+                .transfer(100_000, &mint_keypair, &node_keypair.pubkey())
+                .unwrap();
+            let mut bank_forks = BankForks::new(bank0);
+
+            // Fill bank_forks with banks with votes landing in the next slot
+            // Create enough banks such that vote account will root slots 0 and 1
+            for x in 0..33 {
+                let previous_bank = bank_forks.get(x).unwrap();
+                let bank = Bank::new_from_parent(previous_bank, &Pubkey::default(), x + 1);
+                let vote = vote_transaction::new_vote_transaction(
+                    vec![x],
+                    previous_bank.hash(),
+                    previous_bank.last_blockhash(),
+                    &node_keypair,
+                    &vote_keypair,
+                    &vote_keypair,
+                );
+                bank.process_transaction(&vote).unwrap();
+                bank_forks.insert(bank);
+            }
+
+            let working_bank = bank_forks.working_bank();
+            let root = get_vote_account_root_slot(vote_keypair.pubkey(), &working_bank);
+            for x in 0..root {
+                bank_forks.set_root(x, &None, None);
+            }
+
+            // Add an additional bank/vote that will root slot 2
+            let bank33 = bank_forks.get(33).unwrap();
+            let bank34 = Bank::new_from_parent(bank33, &Pubkey::default(), 34);
+            let vote33 = vote_transaction::new_vote_transaction(
+                vec![33],
+                bank33.hash(),
+                bank33.last_blockhash(),
+                &node_keypair,
+                &vote_keypair,
+                &vote_keypair,
+            );
+            bank34.process_transaction(&vote33).unwrap();
+            bank_forks.insert(bank34);
+
+            let working_bank = bank_forks.working_bank();
+            let root = get_vote_account_root_slot(vote_keypair.pubkey(), &working_bank);
+            let ancestors = working_bank.status_cache_ancestors();
+            let _ = AggregateCommitmentService::update_commitment_cache(
+                &block_commitment_cache,
+                CommitmentAggregationData {
+                    bank: working_bank,
+                    root: 0,
+                    total_staked: 100,
+                },
+                ancestors,
+            );
+            let largest_confirmed_root = block_commitment_cache
+                .read()
+                .unwrap()
+                .largest_confirmed_root();
+            bank_forks.set_root(root, &None, Some(largest_confirmed_root));
+            let largest_confirmed_root_bank = bank_forks.get(largest_confirmed_root);
+            assert!(largest_confirmed_root_bank.is_some());
+
+            // Add a forked bank. Because the vote for bank 33 landed in the non-ancestor, the vote
+            // account's root (and thus the highest_confirmed_root) rolls back to slot 1
+            let bank33 = bank_forks.get(33).unwrap();
+            let bank35 = Bank::new_from_parent(bank33, &Pubkey::default(), 35);
+            bank_forks.insert(bank35);
+
+            let working_bank = bank_forks.working_bank();
+            let ancestors = working_bank.status_cache_ancestors();
+            let _ = AggregateCommitmentService::update_commitment_cache(
+                &block_commitment_cache,
+                CommitmentAggregationData {
+                    bank: working_bank,
+                    root: 1,
+                    total_staked: 100,
+                },
+                ancestors,
+            );
+            let largest_confirmed_root = block_commitment_cache
+                .read()
+                .unwrap()
+                .largest_confirmed_root();
+            let largest_confirmed_root_bank = bank_forks.get(largest_confirmed_root);
+            assert!(largest_confirmed_root_bank.is_some());
+
+            // Add additional banks beyond lockout built on the new fork to ensure that behavior
+            // continues normally
+            for x in 35..=37 {
+                let previous_bank = bank_forks.get(x).unwrap();
+                let bank = Bank::new_from_parent(previous_bank, &Pubkey::default(), x + 1);
+                let vote = vote_transaction::new_vote_transaction(
+                    vec![x],
+                    previous_bank.hash(),
+                    previous_bank.last_blockhash(),
+                    &node_keypair,
+                    &vote_keypair,
+                    &vote_keypair,
+                );
+                bank.process_transaction(&vote).unwrap();
+                bank_forks.insert(bank);
+            }
+
+            let working_bank = bank_forks.working_bank();
+            let root = get_vote_account_root_slot(vote_keypair.pubkey(), &working_bank);
+            let ancestors = working_bank.status_cache_ancestors();
+            let _ = AggregateCommitmentService::update_commitment_cache(
+                &block_commitment_cache,
+                CommitmentAggregationData {
+                    bank: working_bank,
+                    root: 0,
+                    total_staked: 100,
+                },
+                ancestors,
+            );
+            let largest_confirmed_root = block_commitment_cache
+                .read()
+                .unwrap()
+                .largest_confirmed_root();
+            bank_forks.set_root(root, &None, Some(largest_confirmed_root));
+            let largest_confirmed_root_bank = bank_forks.get(largest_confirmed_root);
+            assert!(largest_confirmed_root_bank.is_some());
+        }
+        Blockstore::destroy(&ledger_path).unwrap();
+    }
 }

core/src/crds_gossip.rs

@@ -159,8 +159,9 @@ impl CrdsGossip {
     pub fn generate_pull_responses(
         &self,
         filters: &[(CrdsValue, CrdsFilter)],
+        now: u64,
     ) -> Vec<Vec<CrdsValue>> {
-        self.pull.generate_pull_responses(&self.crds, filters)
+        self.pull.generate_pull_responses(&self.crds, filters, now)
     }
 
     pub fn filter_pull_responses(

core/src/crds_gossip_pull.rs

@@ -131,7 +131,7 @@ pub struct CrdsGossipPull {
     /// timestamp of last request
     pub pull_request_time: HashMap<Pubkey, u64>,
     /// hash and insert time
-    purged_values: VecDeque<(Hash, u64)>,
+    pub purged_values: VecDeque<(Hash, u64)>,
     pub crds_timeout: u64,
     pub msg_timeout: u64,
     pub num_pulls: usize,
@@ -237,8 +237,9 @@ impl CrdsGossipPull {
         &self,
         crds: &Crds,
         requests: &[(CrdsValue, CrdsFilter)],
+        now: u64,
     ) -> Vec<Vec<CrdsValue>> {
-        self.filter_crds_values(crds, requests)
+        self.filter_crds_values(crds, requests, now)
     }
 
     // Checks if responses should be inserted and
@@ -364,22 +365,50 @@ impl CrdsGossipPull {
         for (value_hash, _insert_timestamp) in &self.purged_values {
             filters.iter_mut().for_each(|filter| filter.add(value_hash));
         }
+
         filters
     }
+
     /// filter values that fail the bloom filter up to max_bytes
     fn filter_crds_values(
         &self,
         crds: &Crds,
         filters: &[(CrdsValue, CrdsFilter)],
+        now: u64,
     ) -> Vec<Vec<CrdsValue>> {
         let mut ret = vec![vec![]; filters.len()];
+        let msg_timeout = CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS;
+        let jitter = rand::thread_rng().gen_range(0, msg_timeout / 4);
+        let start = filters.len();
+        //skip filters from callers that are too old
+        let future = now.saturating_add(msg_timeout);
+        let past = now.saturating_sub(msg_timeout);
+        let recent: Vec<_> = filters
+            .iter()
+            .filter(|(caller, _)| caller.wallclock() < future && caller.wallclock() >= past)
+            .collect();
+        inc_new_counter_info!(
+            "gossip_filter_crds_values-dropped_requests",
+            start - recent.len()
+        );
+        if recent.is_empty() {
+            return ret;
+        }
+        let mut total_skipped = 0;
         for v in crds.table.values() {
-            filters.iter().enumerate().for_each(|(i, (_, filter))| {
+            recent.iter().enumerate().for_each(|(i, (caller, filter))| {
+                //skip values that are too new
+                if v.value.wallclock() > caller.wallclock().checked_add(jitter).unwrap_or_else(|| 0)
+                {
+                    total_skipped += 1;
+                    return;
+                }
                 if !filter.contains(&v.value_hash) {
                     ret[i].push(v.value.clone());
                 }
             });
         }
+        inc_new_counter_info!("gossip_filter_crds_values-dropped_values", total_skipped);
         ret
     }
     pub fn make_timeouts_def(
@@ -636,6 +665,62 @@ mod test {
         }
     }
 
+    #[test]
+    fn test_generate_pull_responses() {
+        let mut node_crds = Crds::default();
+        let entry = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
+            &Pubkey::new_rand(),
+            0,
+        )));
+        let node_pubkey = entry.label().pubkey();
+        let node = CrdsGossipPull::default();
+        node_crds.insert(entry, 0).unwrap();
+        let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
+            &Pubkey::new_rand(),
+            0,
+        )));
+        node_crds.insert(new, 0).unwrap();
+        let req = node.new_pull_request(
+            &node_crds,
+            &node_pubkey,
+            0,
+            0,
+            &HashMap::new(),
+            PACKET_DATA_SIZE,
+        );
+
+        let mut dest_crds = Crds::default();
+        let dest = CrdsGossipPull::default();
+        let (_, filters, caller) = req.unwrap();
+        let mut filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
+        let rsp = dest.generate_pull_responses(&dest_crds, &filters, 0);
+
+        assert_eq!(rsp[0].len(), 0);
+
+        let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
+            &Pubkey::new_rand(),
+            CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS,
+        )));
+        dest_crds
+            .insert(new, CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS)
+            .unwrap();
+
+        //should skip new value since caller is to old
+        let rsp =
+            dest.generate_pull_responses(&dest_crds, &filters, CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS);
+        assert_eq!(rsp[0].len(), 0);
+
+        //should return new value since caller is new
+        filters[0].0 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
+            &Pubkey::new_rand(),
+            CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS + 1,
+        )));
+
+        let rsp =
+            dest.generate_pull_responses(&dest_crds, &filters, CRDS_GOSSIP_PULL_MSG_TIMEOUT_MS);
+        assert_eq!(rsp[0].len(), 1);
+    }
+
     #[test]
     fn test_process_pull_request() {
         let mut node_crds = Crds::default();
@@ -664,7 +749,7 @@ mod test {
         let mut dest = CrdsGossipPull::default();
         let (_, filters, caller) = req.unwrap();
         let filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
-        let rsp = dest.generate_pull_responses(&dest_crds, &filters);
+        let rsp = dest.generate_pull_responses(&dest_crds, &filters, 0);
         dest.process_pull_requests(&mut dest_crds, filters, 1);
         assert!(rsp.iter().all(|rsp| rsp.is_empty()));
         assert!(dest_crds.lookup(&caller.label()).is_some());
@@ -688,7 +773,7 @@ mod test {
         let mut node_crds = Crds::default();
         let entry = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
             &Pubkey::new_rand(),
-            0,
+            1,
         )));
         let node_pubkey = entry.label().pubkey();
         let mut node = CrdsGossipPull::default();
@@ -696,7 +781,7 @@ mod test {
 
         let new = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
             &Pubkey::new_rand(),
-            0,
+            1,
         )));
         node_crds.insert(new, 0).unwrap();
 
@@ -735,7 +820,7 @@ mod test {
             );
             let (_, filters, caller) = req.unwrap();
             let filters: Vec<_> = filters.into_iter().map(|f| (caller.clone(), f)).collect();
-            let mut rsp = dest.generate_pull_responses(&dest_crds, &filters);
+            let mut rsp = dest.generate_pull_responses(&dest_crds, &filters, 0);
             dest.process_pull_requests(&mut dest_crds, filters, 0);
             // if there is a false positive this is empty
             // prob should be around 0.1 per iteration

core/src/lib.rs

@@ -19,6 +19,7 @@ pub mod contact_info;
 pub mod bank_weight_fork_choice;
 pub mod cluster_info;
 pub mod cluster_slots;
+pub mod cluster_slots_service;
 pub mod consensus;
 pub mod crds;
 pub mod crds_gossip;

core/src/repair_service.rs

@@ -13,7 +13,7 @@ use crate::{
 use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
 use solana_ledger::{
     bank_forks::BankForks,
-    blockstore::{Blockstore, CompletedSlotsReceiver, SlotMeta},
+    blockstore::{Blockstore, SlotMeta},
     shred::Nonce,
 };
 use solana_measure::measure::Measure;
@@ -80,8 +80,6 @@ pub struct RepairTiming {
     pub set_root_elapsed: u64,
     pub get_votes_elapsed: u64,
     pub add_votes_elapsed: u64,
-    pub lowest_slot_elapsed: u64,
-    pub update_completed_slots_elapsed: u64,
     pub get_best_orphans_elapsed: u64,
     pub get_best_shreds_elapsed: u64,
     pub send_repairs_elapsed: u64,
@@ -93,15 +91,11 @@ impl RepairTiming {
         set_root_elapsed: u64,
         get_votes_elapsed: u64,
         add_votes_elapsed: u64,
-        lowest_slot_elapsed: u64,
-        update_completed_slots_elapsed: u64,
         send_repairs_elapsed: u64,
     ) {
         self.set_root_elapsed += set_root_elapsed;
         self.get_votes_elapsed += get_votes_elapsed;
         self.add_votes_elapsed += add_votes_elapsed;
-        self.lowest_slot_elapsed += lowest_slot_elapsed;
-        self.update_completed_slots_elapsed += update_completed_slots_elapsed;
         self.send_repairs_elapsed += send_repairs_elapsed;
     }
 }
@@ -114,7 +108,6 @@ pub const MAX_ORPHANS: usize = 5;
 
 pub struct RepairInfo {
     pub bank_forks: Arc<RwLock<BankForks>>,
-    pub completed_slots_receiver: CompletedSlotsReceiver,
     pub epoch_schedule: EpochSchedule,
     pub duplicate_slots_reset_sender: DuplicateSlotsResetSender,
 }
@@ -183,18 +176,12 @@ impl RepairService {
         let mut repair_weight = RepairWeight::new(repair_info.bank_forks.read().unwrap().root());
         let serve_repair = ServeRepair::new(cluster_info.clone());
         let id = cluster_info.id();
-        Self::initialize_lowest_slot(id, blockstore, &cluster_info);
         let mut repair_stats = RepairStats::default();
         let mut repair_timing = RepairTiming::default();
         let mut last_stats = Instant::now();
         let duplicate_slot_repair_statuses: HashMap<Slot, DuplicateSlotRepairStatus> =
             HashMap::new();
 
-        Self::initialize_epoch_slots(
-            blockstore,
-            &cluster_info,
-            &repair_info.completed_slots_receiver,
-        );
         loop {
             if exit.load(Ordering::Relaxed) {
                 break;
@@ -203,8 +190,6 @@ impl RepairService {
             let mut set_root_elapsed;
             let mut get_votes_elapsed;
             let mut add_votes_elapsed;
-            let mut lowest_slot_elapsed;
-            let mut update_completed_slots_elapsed;
             let repairs = {
                 let root_bank = repair_info.bank_forks.read().unwrap().root_bank().clone();
                 let new_root = root_bank.slot();
@@ -237,15 +222,6 @@ impl RepairService {
                     root_bank.epoch_schedule(),
                 );
                 add_votes_elapsed.stop();
-
-                lowest_slot_elapsed = Measure::start("lowest_slot_elapsed");
-                let lowest_slot = blockstore.lowest_slot();
-                Self::update_lowest_slot(&id, lowest_slot, &cluster_info);
-                lowest_slot_elapsed.stop();
-                update_completed_slots_elapsed = Measure::start("update_completed_slots_elapsed");
-                Self::update_completed_slots(&repair_info.completed_slots_receiver, &cluster_info);
-                cluster_slots.update(new_root, &cluster_info, &repair_info.bank_forks);
-                update_completed_slots_elapsed.stop();
                 /*let new_duplicate_slots = Self::find_new_duplicate_slots(
                     &duplicate_slot_repair_statuses,
                     blockstore,
@@ -301,8 +277,6 @@ impl RepairService {
                 set_root_elapsed.as_us(),
                 get_votes_elapsed.as_us(),
                 add_votes_elapsed.as_us(),
-                lowest_slot_elapsed.as_us(),
-                update_completed_slots_elapsed.as_us(),
                 send_repairs_elapsed.as_us(),
             );
 
@@ -337,16 +311,6 @@ impl RepairService {
                         repair_timing.get_best_shreds_elapsed,
                         i64
                     ),
-                    (
-                        "lowest-slot-elapsed",
-                        repair_timing.lowest_slot_elapsed,
-                        i64
-                    ),
-                    (
-                        "update-completed-slots-elapsed",
-                        repair_timing.update_completed_slots_elapsed,
-                        i64
-                    ),
                     (
                         "send-repairs-elapsed",
                         repair_timing.send_repairs_elapsed,
@@ -652,59 +616,6 @@ impl RepairService {
             .collect()
     }
 
-    fn initialize_lowest_slot(id: Pubkey, blockstore: &Blockstore, cluster_info: &ClusterInfo) {
-        // Safe to set into gossip because by this time, the leader schedule cache should
-        // also be updated with the latest root (done in blockstore_processor) and thus
-        // will provide a schedule to window_service for any incoming shreds up to the
-        // last_confirmed_epoch.
-        cluster_info.push_lowest_slot(id, blockstore.lowest_slot());
-    }
-
-    fn update_completed_slots(
-        completed_slots_receiver: &CompletedSlotsReceiver,
-        cluster_info: &ClusterInfo,
-    ) {
-        let mut slots: Vec<Slot> = vec![];
-        while let Ok(mut more) = completed_slots_receiver.try_recv() {
-            slots.append(&mut more);
-        }
-        slots.sort();
-        if !slots.is_empty() {
-            cluster_info.push_epoch_slots(&slots);
-        }
-    }
-
-    fn update_lowest_slot(id: &Pubkey, lowest_slot: Slot, cluster_info: &ClusterInfo) {
-        cluster_info.push_lowest_slot(*id, lowest_slot);
-    }
-
-    fn initialize_epoch_slots(
-        blockstore: &Blockstore,
-        cluster_info: &ClusterInfo,
-        completed_slots_receiver: &CompletedSlotsReceiver,
-    ) {
-        let root = blockstore.last_root();
-        let mut slots: Vec<_> = blockstore
-            .live_slots_iterator(root)
-            .filter_map(|(slot, slot_meta)| {
-                if slot_meta.is_full() {
-                    Some(slot)
-                } else {
-                    None
-                }
-            })
-            .collect();
-
-        while let Ok(mut more) = completed_slots_receiver.try_recv() {
-            slots.append(&mut more);
-        }
-        slots.sort();
-        slots.dedup();
-        if !slots.is_empty() {
-            cluster_info.push_epoch_slots(&slots);
-        }
-    }
-
     pub fn join(self) -> thread::Result<()> {
         self.t_repair.join()
     }
@@ -982,19 +893,6 @@ mod test {
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }
 
-    #[test]
-    pub fn test_update_lowest_slot() {
-        let node_info = Node::new_localhost_with_pubkey(&Pubkey::default());
-        let cluster_info = ClusterInfo::new_with_invalid_keypair(node_info.info);
-        RepairService::update_lowest_slot(&Pubkey::default(), 5, &cluster_info);
-        let lowest = cluster_info
-            .get_lowest_slot_for_node(&Pubkey::default(), None, |lowest_slot, _| {
-                lowest_slot.clone()
-            })
-            .unwrap();
-        assert_eq!(lowest.lowest, 5);
-    }
-
     #[test]
     pub fn test_generate_duplicate_repairs_for_slot() {
         let blockstore_path = get_tmp_ledger_path!();

core/src/repair_weight.rs

@@ -170,11 +170,20 @@ impl RepairWeight {
         if new_root == self.root {
             return;
         }
+
+        // Root slot of the tree that contains `new_root`, if one exists
+        let new_root_tree_root = self.slot_to_tree.get(&new_root).cloned();
+
         // Purge outdated trees from `self.trees`
         let subtrees_to_purge: Vec<_> = self
             .trees
             .keys()
-            .filter(|subtree_root| **subtree_root < new_root && **subtree_root != self.root)
+            .filter(|subtree_root| {
+                **subtree_root < new_root
+                    && new_root_tree_root
+                        .map(|new_root_tree_root| **subtree_root != new_root_tree_root)
+                        .unwrap_or(true)
+            })
             .cloned()
             .collect();
         for subtree_root in subtrees_to_purge {
@@ -188,25 +197,26 @@ impl RepairWeight {
             );
         }
 
-        let mut root_tree = self.trees.remove(&self.root).expect("root tree must exist");
+        if let Some(new_root_tree_root) = new_root_tree_root {
+            let mut new_root_tree = self
+                .trees
+                .remove(&new_root_tree_root)
+                .expect("Found slot root earlier in self.slot_to_trees, treee must exist");
+            // Find all descendants of `self.root` that are not reachable from `new_root`.
+            // These are exactly the unrooted slots, which can be purged and added to
+            // `self.unrooted_slots`.
+            let unrooted_slots = new_root_tree.subtree_diff(new_root_tree_root, new_root);
+            self.remove_tree_slots(unrooted_slots.iter(), new_root);
 
-        // Find all descendants of `self.root` that are not reachable from `new_root`.
-        // These are exactly the unrooted slots, which can be purged and added to
-        // `self.unrooted_slots`.
-        let unrooted_slots = root_tree.subtree_diff(self.root, new_root);
-        self.remove_tree_slots(unrooted_slots.iter(), new_root);
+            new_root_tree.set_root(new_root);
 
-        if !root_tree.contains_slot(new_root) {
-            // If the current `root_tree` does not contain the new root, we can
-            // just make a new tree for the new root
-            self.insert_new_tree(new_root);
-        } else {
-            root_tree.set_root(new_root);
             // Update `self.slot_to_tree` to reflect new root
-            self.rename_tree_root(&root_tree, new_root);
+            self.rename_tree_root(&new_root_tree, new_root);
 
             // Insert the tree for the new root
-            self.trees.insert(new_root, root_tree);
+            self.trees.insert(new_root, new_root_tree);
+        } else {
+            self.insert_new_tree(new_root);
         }
 
         // Purge `self.unrooted_slots` of slots less than `new_root` as we know any
@@ -954,6 +964,27 @@ mod test {
         }
     }
 
+    #[test]
+    fn test_set_root_existing_non_root_tree() {
+        let (_, _, mut repair_weight) = setup_orphan_repair_weight();
+
+        // Set root in an existing orphan branch, slot 10
+        repair_weight.set_root(10);
+        check_old_root_purged_verify_new_root(0, 10, &repair_weight);
+
+        // Should purge old root tree [0, 6]
+        for slot in 0..6 {
+            assert!(!repair_weight.slot_to_tree.contains_key(&slot));
+        }
+
+        // Should purge orphan parent as well
+        assert!(!repair_weight.slot_to_tree.contains_key(&8));
+
+        // Other higher orphan branch rooted at slot `20` remains unchanged
+        assert_eq!(repair_weight.trees.get(&20).unwrap().root(), 20);
+        assert_eq!(*repair_weight.slot_to_tree.get(&20).unwrap(), 20);
+    }
+
     #[test]
     fn test_set_root_check_unrooted_slots() {
         let (blockstore, bank, mut repair_weight) = setup_orphan_repair_weight();

core/src/retransmit_stage.rs

@@ -4,6 +4,7 @@ use crate::{
     cluster_info::{compute_retransmit_peers, ClusterInfo, DATA_PLANE_FANOUT},
     cluster_info_vote_listener::VerifiedVoteReceiver,
     cluster_slots::ClusterSlots,
+    cluster_slots_service::ClusterSlotsService,
     contact_info::ContactInfo,
     repair_service::DuplicateSlotsResetSender,
     repair_service::RepairInfo,
@@ -394,6 +395,7 @@ pub fn retransmitter(
 pub struct RetransmitStage {
     thread_hdls: Vec<JoinHandle<()>>,
     window_service: WindowService,
+    cluster_slots_service: ClusterSlotsService,
 }
 
 impl RetransmitStage {
@@ -427,9 +429,16 @@ impl RetransmitStage {
             retransmit_receiver,
         );
 
+        let cluster_slots_service = ClusterSlotsService::new(
+            blockstore.clone(),
+            cluster_slots.clone(),
+            bank_forks.clone(),
+            cluster_info.clone(),
+            completed_slots_receiver,
+            exit.clone(),
+        );
         let repair_info = RepairInfo {
             bank_forks,
-            completed_slots_receiver,
             epoch_schedule,
             duplicate_slots_reset_sender,
         };
@@ -466,6 +475,7 @@ impl RetransmitStage {
         Self {
             thread_hdls,
             window_service,
+            cluster_slots_service,
         }
     }
 
@@ -474,6 +484,7 @@ impl RetransmitStage {
             thread_hdl.join()?;
         }
         self.window_service.join()?;
+        self.cluster_slots_service.join()?;
         Ok(())
     }
 }

core/src/rpc.rs

@@ -11,10 +11,14 @@ use crate::{
     validator::ValidatorExit,
 };
 use bincode::serialize;
-use jsonrpc_core::{Error, Metadata, Result};
+use jsonrpc_core::{types::error, Error, Metadata, Result};
 use jsonrpc_derive::rpc;
 use solana_account_decoder::{
-    parse_token::{spl_token_id_v1_0, spl_token_v1_0_native_mint},
+    parse_account_data::AccountAdditionalData,
+    parse_token::{
+        get_token_account_mint, spl_token_id_v1_0, spl_token_v1_0_native_mint,
+        token_amount_to_ui_amount, UiTokenAmount,
+    },
     UiAccount, UiAccountEncoding,
 };
 use solana_client::{
@@ -64,6 +68,7 @@ use std::{
     str::FromStr,
     sync::{Arc, RwLock},
 };
+use tokio::runtime;
 
 fn new_response<T>(bank: &Bank, value: T) -> Result<RpcResponse<T>> {
     let context = RpcResponseContext { slot: bank.slot() };
@@ -78,6 +83,7 @@ pub struct JsonRpcConfig {
     pub identity_pubkey: Pubkey,
     pub faucet_addr: Option<SocketAddr>,
     pub health_check_slot_distance: u64,
+    pub enable_bigtable_ledger_storage: bool,
 }
 
 #[derive(Clone)]
@@ -91,6 +97,8 @@ pub struct JsonRpcRequestProcessor {
     cluster_info: Arc<ClusterInfo>,
     genesis_hash: Hash,
     send_transaction_service: Arc<SendTransactionService>,
+    runtime_handle: runtime::Handle,
+    bigtable_ledger_storage: Option<solana_storage_bigtable::LedgerStorage>,
 }
 impl Metadata for JsonRpcRequestProcessor {}
 
@@ -145,6 +153,7 @@ impl JsonRpcRequestProcessor {
         }
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         config: JsonRpcConfig,
         bank_forks: Arc<RwLock<BankForks>>,
@@ -155,6 +164,8 @@ impl JsonRpcRequestProcessor {
         cluster_info: Arc<ClusterInfo>,
         genesis_hash: Hash,
         send_transaction_service: Arc<SendTransactionService>,
+        runtime: &runtime::Runtime,
+        bigtable_ledger_storage: Option<solana_storage_bigtable::LedgerStorage>,
     ) -> Self {
         Self {
             config,
@@ -166,6 +177,8 @@ impl JsonRpcRequestProcessor {
             cluster_info,
             genesis_hash,
             send_transaction_service,
+            runtime_handle: runtime.handle().clone(),
+            bigtable_ledger_storage,
         }
     }
 
@@ -177,11 +190,30 @@ impl JsonRpcRequestProcessor {
         let config = config.unwrap_or_default();
         let bank = self.bank(config.commitment)?;
         let encoding = config.encoding.unwrap_or(UiAccountEncoding::Binary);
-        new_response(
-            &bank,
-            bank.get_account(pubkey)
-                .map(|account| UiAccount::encode(account, encoding)),
-        )
+        check_slice_and_encoding(&encoding, config.data_slice.is_some())?;
+        let mut response = None;
+        if let Some(account) = bank.get_account(pubkey) {
+            if account.owner == spl_token_id_v1_0() && encoding == UiAccountEncoding::JsonParsed {
+                response = Some(get_parsed_token_account(bank.clone(), pubkey, account));
+            } else if encoding == UiAccountEncoding::Binary && account.data.len() > 128 {
+                let message = "Encoded binary (base 58) data should be less than 128 bytes, please use Binary64 encoding.".to_string();
+                return Err(error::Error {
+                    code: error::ErrorCode::InvalidRequest,
+                    message,
+                    data: None,
+                });
+            } else {
+                response = Some(UiAccount::encode(
+                    pubkey,
+                    account,
+                    encoding,
+                    None,
+                    config.data_slice,
+                ));
+            }
+        }
+
+        new_response(&bank, response)
     }
 
     pub fn get_minimum_balance_for_rent_exemption(
@@ -203,12 +235,27 @@ impl JsonRpcRequestProcessor {
         let config = config.unwrap_or_default();
         let bank = self.bank(config.commitment)?;
         let encoding = config.encoding.unwrap_or(UiAccountEncoding::Binary);
-        Ok(get_filtered_program_accounts(&bank, program_id, filters)
-            .map(|(pubkey, account)| RpcKeyedAccount {
-                pubkey: pubkey.to_string(),
-                account: UiAccount::encode(account, encoding.clone()),
-            })
-            .collect())
+        let data_slice_config = config.data_slice;
+        check_slice_and_encoding(&encoding, data_slice_config.is_some())?;
+        let keyed_accounts = get_filtered_program_accounts(&bank, program_id, filters);
+        let result =
+            if program_id == &spl_token_id_v1_0() && encoding == UiAccountEncoding::JsonParsed {
+                get_parsed_token_accounts(bank, keyed_accounts).collect()
+            } else {
+                keyed_accounts
+                    .map(|(pubkey, account)| RpcKeyedAccount {
+                        pubkey: pubkey.to_string(),
+                        account: UiAccount::encode(
+                            &pubkey,
+                            account,
+                            encoding.clone(),
+                            None,
+                            data_slice_config,
+                        ),
+                    })
+                    .collect()
+            };
+        Ok(result)
     }
 
     pub fn get_inflation_governor(
@@ -507,6 +554,7 @@ impl JsonRpcRequestProcessor {
         slot: Slot,
         encoding: Option<UiTransactionEncoding>,
     ) -> Result<Option<ConfirmedBlock>> {
+        let encoding = encoding.unwrap_or(UiTransactionEncoding::Json);
         if self.config.enable_rpc_transaction_history
             && slot
                 <= self
@@ -515,7 +563,15 @@ impl JsonRpcRequestProcessor {
                     .unwrap()
                     .largest_confirmed_root()
         {
-            let result = self.blockstore.get_confirmed_block(slot, encoding);
+            let result = self.blockstore.get_confirmed_block(slot, Some(encoding));
+            if result.is_err() {
+                if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
+                    return Ok(self
+                        .runtime_handle
+                        .block_on(bigtable_ledger_storage.get_confirmed_block(slot, encoding))
+                        .ok());
+                }
+            }
             self.check_slot_cleaned_up(&result, slot)?;
             Ok(result.ok())
         } else {
@@ -544,9 +600,25 @@ impl JsonRpcRequestProcessor {
                 MAX_GET_CONFIRMED_BLOCKS_RANGE
             )));
         }
+
+        let lowest_slot = self.blockstore.lowest_slot();
+        if start_slot < lowest_slot {
+            // If the starting slot is lower than what's available in blockstore assume the entire
+            // [start_slot..end_slot] can be fetched from BigTable.
+            if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
+                return Ok(self
+                    .runtime_handle
+                    .block_on(
+                        bigtable_ledger_storage
+                            .get_confirmed_blocks(start_slot, (end_slot - start_slot) as usize),
+                    )
+                    .unwrap_or_else(|_| vec![]));
+            }
+        }
+
         Ok(self
             .blockstore
-            .rooted_slot_iterator(max(start_slot, self.blockstore.lowest_slot()))
+            .rooted_slot_iterator(max(start_slot, lowest_slot))
             .map_err(|_| Error::internal_error())?
             .filter(|&slot| slot <= end_slot)
             .collect())
@@ -640,6 +712,16 @@ impl JsonRpcRequestProcessor {
                             err,
                         }
                     })
+                    .or_else(|| {
+                        if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
+                            self.runtime_handle
+                                .block_on(bigtable_ledger_storage.get_signature_status(&signature))
+                                .map(Some)
+                                .unwrap_or(None)
+                        } else {
+                            None
+                        }
+                    })
             } else {
                 None
             };
@@ -681,23 +763,39 @@ impl JsonRpcRequestProcessor {
         &self,
         signature: Signature,
         encoding: Option<UiTransactionEncoding>,
-    ) -> Result<Option<ConfirmedTransaction>> {
+    ) -> Option<ConfirmedTransaction> {
+        let encoding = encoding.unwrap_or(UiTransactionEncoding::Json);
         if self.config.enable_rpc_transaction_history {
-            Ok(self
+            match self
                 .blockstore
-                .get_confirmed_transaction(signature, encoding)
+                .get_confirmed_transaction(signature, Some(encoding))
                 .unwrap_or(None)
-                .filter(|confirmed_transaction| {
-                    confirmed_transaction.slot
+            {
+                Some(confirmed_transaction) => {
+                    if confirmed_transaction.slot
                         <= self
                             .block_commitment_cache
                             .read()
                             .unwrap()
-                            .largest_confirmed_root()
-                }))
-        } else {
-            Ok(None)
+                            .highest_confirmed_slot()
+                    {
+                        return Some(confirmed_transaction);
+                    }
+                }
+                None => {
+                    if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
+                        return self
+                            .runtime_handle
+                            .block_on(
+                                bigtable_ledger_storage
+                                    .get_confirmed_transaction(&signature, encoding),
+                            )
+                            .unwrap_or(None);
+                    }
+                }
+            }
         }
+        None
     }
 
     pub fn get_confirmed_signatures_for_address(
@@ -707,6 +805,8 @@ impl JsonRpcRequestProcessor {
         end_slot: Slot,
     ) -> Result<Vec<Signature>> {
         if self.config.enable_rpc_transaction_history {
+            // TODO: Add bigtable_ledger_storage support as a part of
+            // https://github.com/solana-labs/solana/pull/10928
             let end_slot = min(
                 end_slot,
                 self.block_commitment_cache
@@ -724,17 +824,31 @@ impl JsonRpcRequestProcessor {
     }
 
     pub fn get_first_available_block(&self) -> Result<Slot> {
-        Ok(self
+        let slot = self
             .blockstore
             .get_first_available_block()
-            .unwrap_or_default())
+            .unwrap_or_default();
+
+        if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
+            let bigtable_slot = self
+                .runtime_handle
+                .block_on(bigtable_ledger_storage.get_first_available_block())
+                .unwrap_or(None)
+                .unwrap_or(slot);
+
+            if bigtable_slot < slot {
+                return Ok(bigtable_slot);
+            }
+        }
+        Ok(slot)
     }
 
     pub fn get_confirmed_signatures_for_address2(
         &self,
         address: Pubkey,
-        before: Option<Signature>,
-        limit: usize,
+        mut before: Option<Signature>,
+        until: Option<Signature>,
+        mut limit: usize,
     ) -> Result<Vec<RpcConfirmedTransactionStatusWithSignature>> {
         if self.config.enable_rpc_transaction_history {
             let highest_confirmed_root = self
@@ -743,16 +857,43 @@ impl JsonRpcRequestProcessor {
                 .unwrap()
                 .highest_confirmed_slot();
 
-            let results = self
+            let mut results = self
                 .blockstore
                 .get_confirmed_signatures_for_address2(
                     address,
                     highest_confirmed_root,
                     before,
+                    until,
                     limit,
                 )
                 .map_err(|err| Error::invalid_params(format!("{}", err)))?;
 
+            if results.len() < limit {
+                if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
+                    if !results.is_empty() {
+                        limit -= results.len();
+                        before = results.last().map(|x| x.signature);
+                    }
+
+                    let bigtable_results = self.runtime_handle.block_on(
+                        bigtable_ledger_storage.get_confirmed_signatures_for_address(
+                            &address,
+                            before.as_ref(),
+                            until.as_ref(),
+                            limit,
+                        ),
+                    );
+                    match bigtable_results {
+                        Ok(mut bigtable_results) => {
+                            results.append(&mut bigtable_results);
+                        }
+                        Err(err) => {
+                            warn!("{:?}", err);
+                        }
+                    }
+                }
+            }
+
             Ok(results.into_iter().map(|x| x.into()).collect())
         } else {
             Ok(vec![])
@@ -824,7 +965,7 @@ impl JsonRpcRequestProcessor {
         &self,
         pubkey: &Pubkey,
         commitment: Option<CommitmentConfig>,
-    ) -> Result<RpcResponse<RpcTokenAmount>> {
+    ) -> Result<RpcResponse<UiTokenAmount>> {
         let bank = self.bank(commitment)?;
         let account = bank.get_account(pubkey).ok_or_else(|| {
             Error::invalid_params("Invalid param: could not find account".to_string())
@@ -851,7 +992,7 @@ impl JsonRpcRequestProcessor {
         &self,
         mint: &Pubkey,
         commitment: Option<CommitmentConfig>,
-    ) -> Result<RpcResponse<RpcTokenAmount>> {
+    ) -> Result<RpcResponse<UiTokenAmount>> {
         let bank = self.bank(commitment)?;
         let (mint_owner, decimals) = get_mint_owner_and_decimals(&bank, mint)?;
         if mint_owner != spl_token_id_v1_0() {
@@ -939,6 +1080,8 @@ impl JsonRpcRequestProcessor {
         let config = config.unwrap_or_default();
         let bank = self.bank(config.commitment)?;
         let encoding = config.encoding.unwrap_or(UiAccountEncoding::Binary);
+        let data_slice_config = config.data_slice;
+        check_slice_and_encoding(&encoding, data_slice_config.is_some())?;
         let (token_program_id, mint) = get_token_program_id_and_mint(&bank, token_account_filter)?;
 
         let mut filters = vec![
@@ -959,12 +1102,23 @@ impl JsonRpcRequestProcessor {
                 encoding: None,
             }));
         }
-        let accounts = get_filtered_program_accounts(&bank, &token_program_id, filters)
-            .map(|(pubkey, account)| RpcKeyedAccount {
-                pubkey: pubkey.to_string(),
-                account: UiAccount::encode(account, encoding.clone()),
-            })
-            .collect();
+        let keyed_accounts = get_filtered_program_accounts(&bank, &token_program_id, filters);
+        let accounts = if encoding == UiAccountEncoding::JsonParsed {
+            get_parsed_token_accounts(bank.clone(), keyed_accounts).collect()
+        } else {
+            keyed_accounts
+                .map(|(pubkey, account)| RpcKeyedAccount {
+                    pubkey: pubkey.to_string(),
+                    account: UiAccount::encode(
+                        &pubkey,
+                        account,
+                        encoding.clone(),
+                        None,
+                        data_slice_config,
+                    ),
+                })
+                .collect()
+        };
         new_response(&bank, accounts)
     }
 
@@ -977,6 +1131,8 @@ impl JsonRpcRequestProcessor {
         let config = config.unwrap_or_default();
         let bank = self.bank(config.commitment)?;
         let encoding = config.encoding.unwrap_or(UiAccountEncoding::Binary);
+        let data_slice_config = config.data_slice;
+        check_slice_and_encoding(&encoding, data_slice_config.is_some())?;
         let (token_program_id, mint) = get_token_program_id_and_mint(&bank, token_account_filter)?;
 
         let mut filters = vec![
@@ -1005,12 +1161,23 @@ impl JsonRpcRequestProcessor {
                 encoding: None,
             }));
         }
-        let accounts = get_filtered_program_accounts(&bank, &token_program_id, filters)
-            .map(|(pubkey, account)| RpcKeyedAccount {
-                pubkey: pubkey.to_string(),
-                account: UiAccount::encode(account, encoding.clone()),
-            })
-            .collect();
+        let keyed_accounts = get_filtered_program_accounts(&bank, &token_program_id, filters);
+        let accounts = if encoding == UiAccountEncoding::JsonParsed {
+            get_parsed_token_accounts(bank.clone(), keyed_accounts).collect()
+        } else {
+            keyed_accounts
+                .map(|(pubkey, account)| RpcKeyedAccount {
+                    pubkey: pubkey.to_string(),
+                    account: UiAccount::encode(
+                        &pubkey,
+                        account,
+                        encoding.clone(),
+                        None,
+                        data_slice_config,
+                    ),
+                })
+                .collect()
+        };
         new_response(&bank, accounts)
     }
 }
@@ -1067,6 +1234,26 @@ fn run_transaction_simulation(
     (executed[0].0.clone().map(|_| ()), log_collector.output())
 }
 
+fn check_slice_and_encoding(encoding: &UiAccountEncoding, data_slice_is_some: bool) -> Result<()> {
+    match encoding {
+        UiAccountEncoding::JsonParsed => {
+            if data_slice_is_some {
+                let message =
+                    "Sliced account data can only be encoded using binary (base 58) or binary64 encoding."
+                        .to_string();
+                Err(error::Error {
+                    code: error::ErrorCode::InvalidRequest,
+                    message,
+                    data: None,
+                })
+            } else {
+                Ok(())
+            }
+        }
+        UiAccountEncoding::Binary | UiAccountEncoding::Binary64 => Ok(()),
+    }
+}
+
 /// Use a set of filters to get an iterator of keyed program accounts from a bank
 fn get_filtered_program_accounts(
     bank: &Arc<Bank>,
@@ -1083,6 +1270,57 @@ fn get_filtered_program_accounts(
         })
 }
 
+pub(crate) fn get_parsed_token_account(
+    bank: Arc<Bank>,
+    pubkey: &Pubkey,
+    account: Account,
+) -> UiAccount {
+    let additional_data = get_token_account_mint(&account.data)
+        .and_then(|mint_pubkey| get_mint_owner_and_decimals(&bank, &mint_pubkey).ok())
+        .map(|(_, decimals)| AccountAdditionalData {
+            spl_token_decimals: Some(decimals),
+        });
+
+    UiAccount::encode(
+        pubkey,
+        account,
+        UiAccountEncoding::JsonParsed,
+        additional_data,
+        None,
+    )
+}
+
+pub(crate) fn get_parsed_token_accounts<I>(
+    bank: Arc<Bank>,
+    keyed_accounts: I,
+) -> impl Iterator<Item = RpcKeyedAccount>
+where
+    I: Iterator<Item = (Pubkey, Account)>,
+{
+    let mut mint_decimals: HashMap<Pubkey, u8> = HashMap::new();
+    keyed_accounts.map(move |(pubkey, account)| {
+        let additional_data = get_token_account_mint(&account.data).map(|mint_pubkey| {
+            let spl_token_decimals = mint_decimals.get(&mint_pubkey).cloned().or_else(|| {
+                let (_, decimals) = get_mint_owner_and_decimals(&bank, &mint_pubkey).ok()?;
+                mint_decimals.insert(mint_pubkey, decimals);
+                Some(decimals)
+            });
+            AccountAdditionalData { spl_token_decimals }
+        });
+
+        RpcKeyedAccount {
+            pubkey: pubkey.to_string(),
+            account: UiAccount::encode(
+                &pubkey,
+                account,
+                UiAccountEncoding::JsonParsed,
+                additional_data,
+                None,
+            ),
+        }
+    })
+}
+
 /// Analyze a passed Pubkey that may be a Token program id or Mint address to determine the program
 /// id and optional Mint
 fn get_token_program_id_and_mint(
@@ -1136,16 +1374,6 @@ fn get_mint_decimals(data: &[u8]) -> Result<u8> {
         .map(|mint: &mut Mint| mint.decimals)
 }
 
-fn token_amount_to_ui_amount(amount: u64, decimals: u8) -> RpcTokenAmount {
-    // Use `amount_to_ui_amount()` once spl_token is bumped to a version that supports it: https://github.com/solana-labs/solana-program-library/pull/211
-    let amount_decimals = amount as f64 / 10_usize.pow(decimals as u32) as f64;
-    RpcTokenAmount {
-        ui_amount: amount_decimals,
-        decimals,
-        amount: amount.to_string(),
-    }
-}
-
 #[rpc]
 pub trait RpcSol {
     type Metadata;
@@ -1437,7 +1665,7 @@ pub trait RpcSol {
         meta: Self::Metadata,
         pubkey_str: String,
         commitment: Option<CommitmentConfig>,
-    ) -> Result<RpcResponse<RpcTokenAmount>>;
+    ) -> Result<RpcResponse<UiTokenAmount>>;
 
     #[rpc(meta, name = "getTokenSupply")]
     fn get_token_supply(
@@ -1445,7 +1673,7 @@ pub trait RpcSol {
         meta: Self::Metadata,
         mint_str: String,
         commitment: Option<CommitmentConfig>,
-    ) -> Result<RpcResponse<RpcTokenAmount>>;
+    ) -> Result<RpcResponse<UiTokenAmount>>;
 
     #[rpc(meta, name = "getTokenLargestAccounts")]
     fn get_token_largest_accounts(
@@ -2008,7 +2236,7 @@ impl RpcSol for RpcSolImpl {
             signature_str
         );
         let signature = verify_signature(&signature_str)?;
-        meta.get_confirmed_transaction(signature, encoding)
+        Ok(meta.get_confirmed_transaction(signature, encoding))
     }
 
     fn get_confirmed_signatures_for_address(
@@ -2058,6 +2286,11 @@ impl RpcSol for RpcSolImpl {
         } else {
             None
         };
+        let until = if let Some(until) = config.until {
+            Some(verify_signature(&until)?)
+        } else {
+            None
+        };
         let limit = config
             .limit
             .unwrap_or(MAX_GET_CONFIRMED_SIGNATURES_FOR_ADDRESS2_LIMIT);
@@ -2069,7 +2302,7 @@ impl RpcSol for RpcSolImpl {
             )));
         }
 
-        meta.get_confirmed_signatures_for_address2(address, before, limit)
+        meta.get_confirmed_signatures_for_address2(address, before, until, limit)
     }
 
     fn get_first_available_block(&self, meta: Self::Metadata) -> Result<Slot> {
@@ -2096,7 +2329,7 @@ impl RpcSol for RpcSolImpl {
         meta: Self::Metadata,
         pubkey_str: String,
         commitment: Option<CommitmentConfig>,
-    ) -> Result<RpcResponse<RpcTokenAmount>> {
+    ) -> Result<RpcResponse<UiTokenAmount>> {
         debug!(
             "get_token_account_balance rpc request received: {:?}",
             pubkey_str
@@ -2110,7 +2343,7 @@ impl RpcSol for RpcSolImpl {
         meta: Self::Metadata,
         mint_str: String,
         commitment: Option<CommitmentConfig>,
-    ) -> Result<RpcResponse<RpcTokenAmount>> {
+    ) -> Result<RpcResponse<UiTokenAmount>> {
         debug!("get_token_supply rpc request received: {:?}", mint_str);
         let mint = verify_pubkey(mint_str)?;
         meta.get_token_supply(&mint, commitment)
@@ -2379,6 +2612,8 @@ pub mod tests {
                 &bank_forks,
                 &exit,
             )),
+            &runtime::Runtime::new().unwrap(),
+            None,
         );
 
         cluster_info.insert_info(ContactInfo::new_with_pubkey_socketaddr(
@@ -2430,6 +2665,8 @@ pub mod tests {
                 &bank_forks,
                 &exit,
             )),
+            &runtime::Runtime::new().unwrap(),
+            None,
         );
         thread::spawn(move || {
             let blockhash = bank.confirmed_last_blockhash().0;
@@ -2824,13 +3061,13 @@ pub mod tests {
     #[test]
     fn test_rpc_get_account_info() {
         let bob_pubkey = Pubkey::new_rand();
-        let RpcHandler { io, meta, .. } = start_rpc_handler_with_tx(&bob_pubkey);
+        let RpcHandler { io, meta, bank, .. } = start_rpc_handler_with_tx(&bob_pubkey);
 
         let req = format!(
             r#"{{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["{}"]}}"#,
             bob_pubkey
         );
-        let res = io.handle_request_sync(&req, meta);
+        let res = io.handle_request_sync(&req, meta.clone());
         let expected = json!({
             "jsonrpc": "2.0",
             "result": {
@@ -2850,6 +3087,54 @@ pub mod tests {
         let result: Response = serde_json::from_str(&res.expect("actual response"))
             .expect("actual response deserialization");
         assert_eq!(expected, result);
+
+        let address = Pubkey::new_rand();
+        let data = vec![1, 2, 3, 4, 5];
+        let mut account = Account::new(42, 5, &Pubkey::default());
+        account.data = data.clone();
+        bank.store_account(&address, &account);
+
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["{}", {{"encoding":"binary64"}}]}}"#,
+            address
+        );
+        let res = io.handle_request_sync(&req, meta.clone());
+        let result: Value = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(result["result"]["value"]["data"], base64::encode(&data));
+
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["{}", {{"encoding":"binary64", "dataSlice": {{"length": 2, "offset": 1}}}}]}}"#,
+            address
+        );
+        let res = io.handle_request_sync(&req, meta.clone());
+        let result: Value = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(
+            result["result"]["value"]["data"],
+            base64::encode(&data[1..3]),
+        );
+
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["{}", {{"encoding":"binary", "dataSlice": {{"length": 2, "offset": 1}}}}]}}"#,
+            address
+        );
+        let res = io.handle_request_sync(&req, meta.clone());
+        let result: Value = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(
+            result["result"]["value"]["data"],
+            bs58::encode(&data[1..3]).into_string(),
+        );
+
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["{}", {{"encoding":"jsonParsed", "dataSlice": {{"length": 2, "offset": 1}}}}]}}"#,
+            address
+        );
+        let res = io.handle_request_sync(&req, meta);
+        let result: Value = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        result["error"].as_object().unwrap();
     }
 
     #[test]
@@ -3526,6 +3811,8 @@ pub mod tests {
                 &bank_forks,
                 &exit,
             )),
+            &runtime::Runtime::new().unwrap(),
+            None,
         );
 
         let req = r#"{"jsonrpc":"2.0","id":1,"method":"sendTransaction","params":["37u9WtQpcm6ULa3Vmu7ySnANv"]}"#;
@@ -3570,6 +3857,8 @@ pub mod tests {
                 &bank_forks,
                 &exit,
             )),
+            &runtime::Runtime::new().unwrap(),
+            None,
         );
 
         let mut bad_transaction =
@@ -3764,6 +4053,8 @@ pub mod tests {
                 &bank_forks,
                 &exit,
             )),
+            &runtime::Runtime::new().unwrap(),
+            None,
         );
         assert_eq!(request_processor.validator_exit(), Ok(false));
         assert_eq!(exit.load(Ordering::Relaxed), false);
@@ -3796,6 +4087,8 @@ pub mod tests {
                 &bank_forks,
                 &exit,
             )),
+            &runtime::Runtime::new().unwrap(),
+            None,
         );
         assert_eq!(request_processor.validator_exit(), Ok(true));
         assert_eq!(exit.load(Ordering::Relaxed), true);
@@ -3887,6 +4180,8 @@ pub mod tests {
                 &bank_forks,
                 &exit,
             )),
+            &runtime::Runtime::new().unwrap(),
+            None,
         );
         assert_eq!(
             request_processor.get_block_commitment(0),
@@ -4424,7 +4719,7 @@ pub mod tests {
         let res = io.handle_request_sync(&req, meta.clone());
         let result: Value = serde_json::from_str(&res.expect("actual response"))
             .expect("actual response deserialization");
-        let balance: RpcTokenAmount =
+        let balance: UiTokenAmount =
             serde_json::from_value(result["result"]["value"].clone()).unwrap();
         let error = f64::EPSILON;
         assert!((balance.ui_amount - 4.2).abs() < error);
@@ -4452,7 +4747,7 @@ pub mod tests {
         let res = io.handle_request_sync(&req, meta.clone());
         let result: Value = serde_json::from_str(&res.expect("actual response"))
             .expect("actual response deserialization");
-        let supply: RpcTokenAmount =
+        let supply: UiTokenAmount =
             serde_json::from_value(result["result"]["value"].clone()).unwrap();
         let error = f64::EPSILON;
         assert!((supply.ui_amount - 2.0 * 4.2).abs() < error);
@@ -4712,7 +5007,7 @@ pub mod tests {
             vec![
                 RpcTokenAccountBalance {
                     address: token_with_different_mint_pubkey.to_string(),
-                    amount: RpcTokenAmount {
+                    amount: UiTokenAmount {
                         ui_amount: 0.42,
                         decimals: 2,
                         amount: "42".to_string(),
@@ -4720,7 +5015,7 @@ pub mod tests {
                 },
                 RpcTokenAccountBalance {
                     address: token_with_smaller_balance.to_string(),
-                    amount: RpcTokenAmount {
+                    amount: UiTokenAmount {
                         ui_amount: 0.1,
                         decimals: 2,
                         amount: "10".to_string(),
@@ -4729,4 +5024,109 @@ pub mod tests {
             ]
         );
     }
+
+    #[test]
+    fn test_token_parsing() {
+        let RpcHandler { io, meta, bank, .. } = start_rpc_handler_with_tx(&Pubkey::new_rand());
+
+        let mut account_data = [0; size_of::<TokenAccount>()];
+        let account: &mut TokenAccount =
+            spl_token_v1_0::state::unpack_unchecked(&mut account_data).unwrap();
+        let mint = SplTokenPubkey::new(&[2; 32]);
+        let owner = SplTokenPubkey::new(&[3; 32]);
+        let delegate = SplTokenPubkey::new(&[4; 32]);
+        *account = TokenAccount {
+            mint,
+            owner,
+            delegate: COption::Some(delegate),
+            amount: 420,
+            is_initialized: true,
+            is_native: false,
+            delegated_amount: 30,
+        };
+        let token_account = Account {
+            lamports: 111,
+            data: account_data.to_vec(),
+            owner: spl_token_id_v1_0(),
+            ..Account::default()
+        };
+        let token_account_pubkey = Pubkey::new_rand();
+        bank.store_account(&token_account_pubkey, &token_account);
+
+        // Add the mint
+        let mut mint_data = [0; size_of::<Mint>()];
+        let mint_state: &mut Mint =
+            spl_token_v1_0::state::unpack_unchecked(&mut mint_data).unwrap();
+        *mint_state = Mint {
+            owner: COption::Some(owner),
+            decimals: 2,
+            is_initialized: true,
+        };
+        let mint_account = Account {
+            lamports: 111,
+            data: mint_data.to_vec(),
+            owner: spl_token_id_v1_0(),
+            ..Account::default()
+        };
+        bank.store_account(&Pubkey::from_str(&mint.to_string()).unwrap(), &mint_account);
+
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["{}", {{"encoding": "jsonParsed"}}]}}"#,
+            token_account_pubkey,
+        );
+        let res = io.handle_request_sync(&req, meta.clone());
+        let result: Value = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(
+            result["result"]["value"]["data"],
+            json!({
+                "program": "spl-token",
+                "space": 120,
+                "parsed": {
+                    "type": "account",
+                    "info": {
+                        "mint": mint.to_string(),
+                        "owner": owner.to_string(),
+                        "tokenAmount": {
+                            "uiAmount": 4.2,
+                            "decimals": 2,
+                            "amount": "420",
+                        },
+                        "delegate": delegate.to_string(),
+                        "isInitialized": true,
+                        "isNative": false,
+                        "delegatedAmount": {
+                            "uiAmount": 0.3,
+                            "decimals": 2,
+                            "amount": "30",
+                        },
+                    }
+                }
+            })
+        );
+
+        // Test Mint
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":["{}", {{"encoding": "jsonParsed"}}]}}"#,
+            mint,
+        );
+        let res = io.handle_request_sync(&req, meta);
+        let result: Value = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(
+            result["result"]["value"]["data"],
+            json!({
+                "program": "spl-token",
+                "space": 40,
+                "parsed": {
+                    "type": "mint",
+                    "info": {
+                        "owner": owner.to_string(),
+                        "decimals": 2,
+                        "isInitialized": true,
+                    }
+                }
+            })
+        );
+    }
 }

core/src/rpc_pubsub.rs

@@ -562,6 +562,7 @@ mod tests {
             Some(RpcAccountInfoConfig {
                 commitment: Some(CommitmentConfig::recent()),
                 encoding: None,
+                data_slice: None,
             }),
         );
 
@@ -601,7 +602,7 @@ mod tests {
                        "lamports": 51,
                        "data": bs58::encode(expected_data).into_string(),
                        "executable": false,
-                       "rentEpoch": 1,
+                       "rentEpoch": 0,
                    },
                },
                "subscription": 0,
@@ -677,6 +678,7 @@ mod tests {
             Some(RpcAccountInfoConfig {
                 commitment: Some(CommitmentConfig::recent()),
                 encoding: Some(UiAccountEncoding::JsonParsed),
+                data_slice: None,
             }),
         );
 
@@ -700,7 +702,13 @@ mod tests {
             .get_account(&nonce_account.pubkey())
             .unwrap()
             .data;
-        let expected_data = parse_account_data(&system_program::id(), &expected_data).unwrap();
+        let expected_data = parse_account_data(
+            &nonce_account.pubkey(),
+            &system_program::id(),
+            &expected_data,
+            None,
+        )
+        .unwrap();
         let expected = json!({
            "jsonrpc": "2.0",
            "method": "accountNotification",
@@ -712,7 +720,7 @@ mod tests {
                        "lamports": 100,
                        "data": expected_data,
                        "executable": false,
-                       "rentEpoch": 1,
+                       "rentEpoch": 0,
                    },
                },
                "subscription": 0,
@@ -800,6 +808,7 @@ mod tests {
             Some(RpcAccountInfoConfig {
                 commitment: Some(CommitmentConfig::root()),
                 encoding: None,
+                data_slice: None,
             }),
         );
 
@@ -856,6 +865,7 @@ mod tests {
             Some(RpcAccountInfoConfig {
                 commitment: Some(CommitmentConfig::root()),
                 encoding: None,
+                data_slice: None,
             }),
         );
 
@@ -889,7 +899,7 @@ mod tests {
                        "lamports": 100,
                        "data": "",
                        "executable": false,
-                       "rentEpoch": 1,
+                       "rentEpoch": 0,
                    },
                },
                "subscription": 0,

core/src/rpc_service.rs

@@ -24,7 +24,7 @@ use std::{
     sync::{mpsc::channel, Arc, RwLock},
     thread::{self, Builder, JoinHandle},
 };
-use tokio::prelude::Future;
+use tokio::runtime;
 
 pub struct JsonRpcService {
     thread_hdl: JoinHandle<()>,
@@ -33,6 +33,7 @@ pub struct JsonRpcService {
     pub request_processor: JsonRpcRequestProcessor, // Used only by test_rpc_new()...
 
     close_handle: Option<CloseHandle>,
+    runtime: runtime::Runtime,
 }
 
 struct RpcRequestMiddleware {
@@ -98,6 +99,9 @@ impl RpcRequestMiddleware {
     }
 
     fn process_file_get(&self, path: &str) -> RequestMiddlewareAction {
+        // Stuck on tokio 0.1 until the jsonrpc-http-server crate upgrades to tokio 0.2
+        use tokio_01::prelude::*;
+
         let stem = path.split_at(1).1; // Drop leading '/' from path
         let filename = {
             match path {
@@ -116,10 +120,10 @@ impl RpcRequestMiddleware {
         RequestMiddlewareAction::Respond {
             should_validate_hosts: true,
             response: Box::new(
-                tokio_fs::file::File::open(filename)
+                tokio_fs_01::file::File::open(filename)
                     .and_then(|file| {
                         let buf: Vec<u8> = Vec::new();
-                        tokio_io::io::read_to_end(file, buf)
+                        tokio_io_01::io::read_to_end(file, buf)
                             .and_then(|item| Ok(hyper::Response::new(item.1.into())))
                             .or_else(|_| Ok(RpcRequestMiddleware::internal_server_error()))
                     })
@@ -256,6 +260,27 @@ impl JsonRpcService {
             &exit_send_transaction_service,
         ));
 
+        let mut runtime = runtime::Builder::new()
+            .threaded_scheduler()
+            .thread_name("rpc-runtime")
+            .enable_all()
+            .build()
+            .expect("Runtime");
+
+        let bigtable_ledger_storage = if config.enable_bigtable_ledger_storage {
+            runtime
+                .block_on(solana_storage_bigtable::LedgerStorage::new(false))
+                .map(|x| {
+                    info!("BigTable ledger storage initialized");
+                    Some(x)
+                })
+                .unwrap_or_else(|err| {
+                    error!("Failed to initialize BigTable ledger storage: {:?}", err);
+                    None
+                })
+        } else {
+            None
+        };
         let request_processor = JsonRpcRequestProcessor::new(
             config,
             bank_forks.clone(),
@@ -266,6 +291,8 @@ impl JsonRpcService {
             cluster_info,
             genesis_hash,
             send_transaction_service,
+            &runtime,
+            bigtable_ledger_storage,
         );
 
         #[cfg(test)]
@@ -325,6 +352,7 @@ impl JsonRpcService {
             .register_exit(Box::new(move || close_handle_.close()));
         Self {
             thread_hdl,
+            runtime,
             #[cfg(test)]
             request_processor: test_request_processor,
             close_handle: Some(close_handle),
@@ -338,6 +366,7 @@ impl JsonRpcService {
     }
 
     pub fn join(self) -> thread::Result<()> {
+        self.runtime.shutdown_background();
         self.thread_hdl.join()
     }
 }

core/src/rpc_subscriptions.rs

@@ -1,6 +1,9 @@
 //! The `pubsub` module implements a threaded subscription service on client RPC request
 
-use crate::commitment::BlockCommitmentCache;
+use crate::{
+    commitment::BlockCommitmentCache,
+    rpc::{get_parsed_token_account, get_parsed_token_accounts},
+};
 use core::hash::Hash;
 use jsonrpc_core::futures::Future;
 use jsonrpc_pubsub::{
@@ -8,7 +11,7 @@ use jsonrpc_pubsub::{
     SubscriptionId,
 };
 use serde::Serialize;
-use solana_account_decoder::{UiAccount, UiAccountEncoding};
+use solana_account_decoder::{parse_token::spl_token_id_v1_0, UiAccount, UiAccountEncoding};
 use solana_client::{
     rpc_config::{RpcAccountInfoConfig, RpcProgramAccountsConfig},
     rpc_filter::RpcFilterType,
@@ -36,7 +39,9 @@ use std::{
     iter,
     sync::{Arc, Mutex, RwLock},
 };
-use tokio::runtime::{Builder as RuntimeBuilder, Runtime, TaskExecutor};
+
+// Stuck on tokio 0.1 until the jsonrpc-pubsub crate upgrades to tokio 0.2
+use tokio_01::runtime::{Builder as RuntimeBuilder, Runtime, TaskExecutor};
 
 const RECEIVE_DELAY_MILLIS: u64 = 100;
 
@@ -184,7 +189,7 @@ where
     K: Eq + Hash + Clone + Copy,
     S: Clone + Serialize,
     B: Fn(&Bank, &K) -> X,
-    F: Fn(X, Slot, Option<T>) -> (Box<dyn Iterator<Item = S>>, Slot),
+    F: Fn(X, &K, Slot, Option<T>, Option<Arc<Bank>>) -> (Box<dyn Iterator<Item = S>>, Slot),
     X: Clone + Serialize + Default,
     T: Clone,
 {
@@ -208,16 +213,19 @@ where
                     cache_slot_info.highest_confirmed_slot
                 }
             };
-            let results = {
-                let bank_forks = bank_forks.read().unwrap();
-                bank_forks
-                    .get(slot)
-                    .map(|desired_bank| bank_method(&desired_bank, hashmap_key))
-                    .unwrap_or_default()
-            };
+            let bank = bank_forks.read().unwrap().get(slot).cloned();
+            let results = bank
+                .clone()
+                .map(|desired_bank| bank_method(&desired_bank, hashmap_key))
+                .unwrap_or_default();
             let mut w_last_notified_slot = last_notified_slot.write().unwrap();
-            let (filter_results, result_slot) =
-                filter_results(results, *w_last_notified_slot, config.as_ref().cloned());
+            let (filter_results, result_slot) = filter_results(
+                results,
+                hashmap_key,
+                *w_last_notified_slot,
+                config.as_ref().cloned(),
+                bank,
+            );
             for result in filter_results {
                 notifier.notify(
                     Response {
@@ -248,18 +256,30 @@ impl RpcNotifier {
 
 fn filter_account_result(
     result: Option<(Account, Slot)>,
+    pubkey: &Pubkey,
     last_notified_slot: Slot,
     encoding: Option<UiAccountEncoding>,
+    bank: Option<Arc<Bank>>,
 ) -> (Box<dyn Iterator<Item = UiAccount>>, Slot) {
     if let Some((account, fork)) = result {
         // If fork < last_notified_slot this means that we last notified for a fork
         // and should notify that the account state has been reverted.
         if fork != last_notified_slot {
             let encoding = encoding.unwrap_or(UiAccountEncoding::Binary);
-            return (
-                Box::new(iter::once(UiAccount::encode(account, encoding))),
-                fork,
-            );
+            if account.owner == spl_token_id_v1_0() && encoding == UiAccountEncoding::JsonParsed {
+                let bank = bank.unwrap(); // If result.is_some(), bank must also be Some
+                return (
+                    Box::new(iter::once(get_parsed_token_account(bank, pubkey, account))),
+                    fork,
+                );
+            } else {
+                return (
+                    Box::new(iter::once(UiAccount::encode(
+                        pubkey, account, encoding, None, None,
+                    ))),
+                    fork,
+                );
+            }
         }
     }
     (Box::new(iter::empty()), last_notified_slot)
@@ -267,8 +287,10 @@ fn filter_account_result(
 
 fn filter_signature_result(
     result: Option<transaction::Result<()>>,
+    _signature: &Signature,
     last_notified_slot: Slot,
     _config: Option<()>,
+    _bank: Option<Arc<Bank>>,
 ) -> (Box<dyn Iterator<Item = RpcSignatureResult>>, Slot) {
     (
         Box::new(
@@ -282,29 +304,33 @@ fn filter_signature_result(
 
 fn filter_program_results(
     accounts: Vec<(Pubkey, Account)>,
+    _program_id: &Pubkey,
     last_notified_slot: Slot,
     config: Option<ProgramConfig>,
+    bank: Option<Arc<Bank>>,
 ) -> (Box<dyn Iterator<Item = RpcKeyedAccount>>, Slot) {
     let config = config.unwrap_or_default();
     let encoding = config.encoding.unwrap_or(UiAccountEncoding::Binary);
     let filters = config.filters;
-    (
-        Box::new(
-            accounts
-                .into_iter()
-                .filter(move |(_, account)| {
-                    filters.iter().all(|filter_type| match filter_type {
-                        RpcFilterType::DataSize(size) => account.data.len() as u64 == *size,
-                        RpcFilterType::Memcmp(compare) => compare.bytes_match(&account.data),
-                    })
-                })
-                .map(move |(pubkey, account)| RpcKeyedAccount {
+    let keyed_accounts = accounts.into_iter().filter(move |(_, account)| {
+        filters.iter().all(|filter_type| match filter_type {
+            RpcFilterType::DataSize(size) => account.data.len() as u64 == *size,
+            RpcFilterType::Memcmp(compare) => compare.bytes_match(&account.data),
+        })
+    });
+    let accounts: Box<dyn Iterator<Item = RpcKeyedAccount>> =
+        if encoding == UiAccountEncoding::JsonParsed {
+            let bank = bank.unwrap(); // If !accounts.is_empty(), bank must be Some
+            Box::new(get_parsed_token_accounts(bank, keyed_accounts))
+        } else {
+            Box::new(
+                keyed_accounts.map(move |(pubkey, account)| RpcKeyedAccount {
                     pubkey: pubkey.to_string(),
-                    account: UiAccount::encode(account, encoding.clone()),
+                    account: UiAccount::encode(&pubkey, account, encoding.clone(), None, None),
                 }),
-        ),
-        last_notified_slot,
-    )
+            )
+        };
+    (accounts, last_notified_slot)
 }
 
 #[derive(Clone)]
@@ -871,7 +897,7 @@ impl RpcSubscriptions {
             &subscriptions.gossip_account_subscriptions,
             &subscriptions.gossip_program_subscriptions,
             &subscriptions.gossip_signature_subscriptions,
-            &bank_forks,
+            bank_forks,
             &cache_slot_info,
             &notifier,
         );
@@ -892,7 +918,7 @@ impl RpcSubscriptions {
         for pubkey in &pubkeys {
             Self::check_account(
                 pubkey,
-                &bank_forks,
+                bank_forks,
                 account_subscriptions.clone(),
                 &notifier,
                 &cache_slot_info,
@@ -906,7 +932,7 @@ impl RpcSubscriptions {
         for program_id in &programs {
             Self::check_program(
                 program_id,
-                &bank_forks,
+                bank_forks,
                 program_subscriptions.clone(),
                 &notifier,
                 &cache_slot_info,
@@ -920,7 +946,7 @@ impl RpcSubscriptions {
         for signature in &signatures {
             Self::check_signature(
                 signature,
-                &bank_forks,
+                bank_forks,
                 signature_subscriptions.clone(),
                 &notifier,
                 &cache_slot_info,
@@ -965,7 +991,7 @@ pub(crate) mod tests {
         system_transaction,
     };
     use std::{fmt::Debug, sync::mpsc::channel, time::Instant};
-    use tokio::{prelude::FutureExt, runtime::Runtime, timer::Delay};
+    use tokio_01::{prelude::FutureExt, runtime::Runtime, timer::Delay};
 
     pub(crate) fn robust_poll_or_panic<T: Debug + Send + 'static>(
         receiver: futures::sync::mpsc::Receiver<T>,
@@ -1030,6 +1056,7 @@ pub(crate) mod tests {
             Some(RpcAccountInfoConfig {
                 commitment: Some(CommitmentConfig::recent()),
                 encoding: None,
+                data_slice: None,
             }),
             sub_id.clone(),
             subscriber,
@@ -1072,7 +1099,7 @@ pub(crate) mod tests {
                        "executable": false,
                        "lamports": 1,
                        "owner": "Budget1111111111111111111111111111111111111",
-                       "rentEpoch": 1,
+                       "rentEpoch": 0,
                     },
                },
                "subscription": 0,
@@ -1158,7 +1185,7 @@ pub(crate) mod tests {
                           "executable": false,
                           "lamports": 1,
                           "owner": "Budget1111111111111111111111111111111111111",
-                          "rentEpoch": 1,
+                          "rentEpoch": 0,
                        },
                        "pubkey": alice.pubkey().to_string(),
                     },
@@ -1526,6 +1553,7 @@ pub(crate) mod tests {
             Some(RpcAccountInfoConfig {
                 commitment: Some(CommitmentConfig::single_gossip()),
                 encoding: None,
+                data_slice: None,
             }),
             sub_id0.clone(),
             subscriber0,
@@ -1579,7 +1607,7 @@ pub(crate) mod tests {
                        "executable": false,
                        "lamports": 1,
                        "owner": "Budget1111111111111111111111111111111111111",
-                       "rentEpoch": 1,
+                       "rentEpoch": 0,
                     },
                },
                "subscription": 0,
@@ -1594,6 +1622,7 @@ pub(crate) mod tests {
             Some(RpcAccountInfoConfig {
                 commitment: Some(CommitmentConfig::single_gossip()),
                 encoding: None,
+                data_slice: None,
             }),
             sub_id1.clone(),
             subscriber1,
@@ -1612,7 +1641,7 @@ pub(crate) mod tests {
                        "executable": false,
                        "lamports": 1,
                        "owner": "Budget1111111111111111111111111111111111111",
-                       "rentEpoch": 1,
+                       "rentEpoch": 0,
                     },
                },
                "subscription": 1,

core/src/tvu.rs

@@ -269,7 +269,7 @@ pub mod tests {
 
         let (blockstore_path, _) = create_new_tmp_ledger!(&genesis_config);
         let (blockstore, l_receiver, completed_slots_receiver) =
-            Blockstore::open_with_signal(&blockstore_path)
+            Blockstore::open_with_signal(&blockstore_path, None)
                 .expect("Expected to successfully open ledger");
         let blockstore = Arc::new(blockstore);
         let bank = bank_forks.working_bank();

core/src/validator.rs

@@ -28,6 +28,7 @@ use solana_ledger::{
     bank_forks::{BankForks, SnapshotConfig},
     bank_forks_utils,
     blockstore::{Blockstore, CompletedSlotsReceiver, PurgeType},
+    blockstore_db::BlockstoreRecoveryMode,
     blockstore_processor::{self, TransactionStatusSender},
     create_new_tmp_ledger,
     hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
@@ -83,6 +84,7 @@ pub struct ValidatorConfig {
     pub no_rocksdb_compaction: bool,
     pub accounts_hash_interval_slots: u64,
     pub max_genesis_archive_unpacked_size: u64,
+    pub wal_recovery_mode: Option<BlockstoreRecoveryMode>,
 }
 
 impl Default for ValidatorConfig {
@@ -110,6 +112,7 @@ impl Default for ValidatorConfig {
             no_rocksdb_compaction: false,
             accounts_hash_interval_slots: std::u64::MAX,
             max_genesis_archive_unpacked_size: MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
+            wal_recovery_mode: None,
         }
     }
 }
@@ -603,7 +606,8 @@ fn new_banks_from_blockstore(
     }
 
     let (mut blockstore, ledger_signal_receiver, completed_slots_receiver) =
-        Blockstore::open_with_signal(blockstore_path).expect("Failed to open ledger database");
+        Blockstore::open_with_signal(blockstore_path, config.wal_recovery_mode.clone())
+            .expect("Failed to open ledger database");
     blockstore.set_no_compaction(config.no_rocksdb_compaction);
 
     let process_options = blockstore_processor::ProcessOptions {

core/tests/bank_forks.rs

@@ -1,31 +1,32 @@
 // Long-running bank_forks tests
 
 macro_rules! DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS {
-    ($x:ident) => {
+    ($x:ident, $y:ident, $z:ident) => {
         #[allow(non_snake_case)]
-        mod $x {
+        mod $z {
             use super::*;
 
             const SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::$x;
+            const OPERATING_MODE: OperatingMode = OperatingMode::$y;
 
             #[test]
             fn test_bank_forks_status_cache_snapshot_n() {
-                run_test_bank_forks_status_cache_snapshot_n(SNAPSHOT_VERSION)
+                run_test_bank_forks_status_cache_snapshot_n(SNAPSHOT_VERSION, OPERATING_MODE)
             }
 
             #[test]
             fn test_bank_forks_snapshot_n() {
-                run_test_bank_forks_snapshot_n(SNAPSHOT_VERSION)
+                run_test_bank_forks_snapshot_n(SNAPSHOT_VERSION, OPERATING_MODE)
             }
 
             #[test]
             fn test_concurrent_snapshot_packaging() {
-                run_test_concurrent_snapshot_packaging(SNAPSHOT_VERSION)
+                run_test_concurrent_snapshot_packaging(SNAPSHOT_VERSION, OPERATING_MODE)
             }
 
             #[test]
             fn test_slots_to_snapshot() {
-                run_test_slots_to_snapshot(SNAPSHOT_VERSION)
+                run_test_slots_to_snapshot(SNAPSHOT_VERSION, OPERATING_MODE)
             }
         }
     };
@@ -52,7 +53,7 @@ mod tests {
     };
     use solana_sdk::{
         clock::Slot,
-        genesis_config::GenesisConfig,
+        genesis_config::{GenesisConfig, OperatingMode},
         hash::hashv,
         pubkey::Pubkey,
         signature::{Keypair, Signer},
@@ -61,8 +62,12 @@ mod tests {
     use std::{fs, path::PathBuf, sync::atomic::AtomicBool, sync::mpsc::channel, sync::Arc};
     use tempfile::TempDir;
 
-    DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_1_0);
-    DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0);
+    DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_1_0, Development, V1_1_0_Development);
+    DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_1_0, Preview, V1_1_0_Preview);
+    DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_1_0, Stable, V1_1_0_Stable);
+    DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0, Development, V1_2_0_Development);
+    DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0, Preview, V1_2_0_Preview);
+    DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0, Stable, V1_2_0_Stable);
 
     struct SnapshotTestConfig {
         accounts_dir: TempDir,
@@ -76,12 +81,14 @@ mod tests {
     impl SnapshotTestConfig {
         fn new(
             snapshot_version: SnapshotVersion,
+            operating_mode: OperatingMode,
             snapshot_interval_slots: u64,
         ) -> SnapshotTestConfig {
             let accounts_dir = TempDir::new().unwrap();
             let snapshot_dir = TempDir::new().unwrap();
             let snapshot_output_path = TempDir::new().unwrap();
-            let genesis_config_info = create_genesis_config(10_000);
+            let mut genesis_config_info = create_genesis_config(10_000);
+            genesis_config_info.genesis_config.operating_mode = operating_mode;
             let bank0 = Bank::new_with_paths(
                 &genesis_config_info.genesis_config,
                 vec![accounts_dir.path().to_path_buf()],
@@ -113,6 +120,7 @@ mod tests {
     fn restore_from_snapshot(
         old_bank_forks: &BankForks,
         old_last_slot: Slot,
+        old_genesis_config: &GenesisConfig,
         account_paths: &[PathBuf],
     ) {
         let (snapshot_path, snapshot_package_output_path) = old_bank_forks
@@ -137,7 +145,7 @@ mod tests {
                 &CompressionType::Bzip2,
             ),
             CompressionType::Bzip2,
-            &GenesisConfig::default(),
+            old_genesis_config,
         )
         .unwrap();
 
@@ -161,6 +169,7 @@ mod tests {
     // `last_slot` bank
     fn run_bank_forks_snapshot_n<F>(
         snapshot_version: SnapshotVersion,
+        operating_mode: OperatingMode,
         last_slot: Slot,
         f: F,
         set_root_interval: u64,
@@ -169,10 +178,9 @@ mod tests {
     {
         solana_logger::setup();
         // Set up snapshotting config
-        let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
+        let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, operating_mode, 1);
 
         let bank_forks = &mut snapshot_test_config.bank_forks;
-        let accounts_dir = &snapshot_test_config.accounts_dir;
         let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair;
 
         let (s, _r) = channel();
@@ -188,6 +196,7 @@ mod tests {
                 bank_forks.set_root(bank.slot(), &sender, None);
             }
         }
+
         // Generate a snapshot package for last bank
         let last_bank = bank_forks.get(last_slot).unwrap();
         let snapshot_config = &snapshot_test_config.snapshot_config;
@@ -206,17 +215,23 @@ mod tests {
             snapshot_version,
         )
         .unwrap();
-
         snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap();
 
-        restore_from_snapshot(bank_forks, last_slot, &[accounts_dir.path().to_path_buf()]);
+        // Restore bank from snapshot
+        let account_paths = &[snapshot_test_config.accounts_dir.path().to_path_buf()];
+        let genesis_config = &snapshot_test_config.genesis_config_info.genesis_config;
+        restore_from_snapshot(bank_forks, last_slot, genesis_config, account_paths);
     }
 
-    fn run_test_bank_forks_snapshot_n(snapshot_version: SnapshotVersion) {
+    fn run_test_bank_forks_snapshot_n(
+        snapshot_version: SnapshotVersion,
+        operating_mode: OperatingMode,
+    ) {
         // create banks upto slot 4 and create 1 new account in each bank. test that bank 4 snapshots
         // and restores correctly
         run_bank_forks_snapshot_n(
             snapshot_version,
+            operating_mode,
             4,
             |bank, mint_keypair| {
                 let key1 = Keypair::new().pubkey();
@@ -247,11 +262,14 @@ mod tests {
         }
     }
 
-    fn run_test_concurrent_snapshot_packaging(snapshot_version: SnapshotVersion) {
+    fn run_test_concurrent_snapshot_packaging(
+        snapshot_version: SnapshotVersion,
+        operating_mode: OperatingMode,
+    ) {
         solana_logger::setup();
 
         // Set up snapshotting config
-        let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1);
+        let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, operating_mode, 1);
 
         let bank_forks = &mut snapshot_test_config.bank_forks;
         let accounts_dir = &snapshot_test_config.accounts_dir;
@@ -396,7 +414,10 @@ mod tests {
         );
     }
 
-    fn run_test_slots_to_snapshot(snapshot_version: SnapshotVersion) {
+    fn run_test_slots_to_snapshot(
+        snapshot_version: SnapshotVersion,
+        operating_mode: OperatingMode,
+    ) {
         solana_logger::setup();
         let num_set_roots = MAX_CACHE_ENTRIES * 2;
 
@@ -405,6 +426,7 @@ mod tests {
             // Make sure this test never clears bank.slots_since_snapshot
             let mut snapshot_test_config = SnapshotTestConfig::new(
                 snapshot_version,
+                operating_mode,
                 (*add_root_interval * num_set_roots * 2) as u64,
             );
             let mut current_bank = snapshot_test_config.bank_forks[0].clone();
@@ -438,7 +460,10 @@ mod tests {
         }
     }
 
-    fn run_test_bank_forks_status_cache_snapshot_n(snapshot_version: SnapshotVersion) {
+    fn run_test_bank_forks_status_cache_snapshot_n(
+        snapshot_version: SnapshotVersion,
+        operating_mode: OperatingMode,
+    ) {
         // create banks upto slot (MAX_CACHE_ENTRIES * 2) + 1 while transferring 1 lamport into 2 different accounts each time
         // this is done to ensure the AccountStorageEntries keep getting cleaned up as the root moves
         // ahead. Also tests the status_cache purge and status cache snapshotting.
@@ -448,6 +473,7 @@ mod tests {
         for set_root_interval in &[1, 4] {
             run_bank_forks_snapshot_n(
                 snapshot_version,
+                operating_mode,
                 (MAX_CACHE_ENTRIES * 2 + 1) as u64,
                 |bank, mint_keypair| {
                     let tx = system_transaction::transfer(

core/tests/crds_gossip.rs

@@ -436,7 +436,7 @@ fn network_run_pull(
                         let rsp = node
                             .lock()
                             .unwrap()
-                            .generate_pull_responses(&filters)
+                            .generate_pull_responses(&filters, now)
                             .into_iter()
                             .flatten()
                             .collect();

core/tests/rpc.rs

@@ -26,7 +26,7 @@ use std::{
     thread::sleep,
     time::{Duration, Instant},
 };
-use tokio::runtime::Runtime;
+use tokio_01::runtime::Runtime;
 
 macro_rules! json_req {
     ($method: expr, $params: expr) => {{
@@ -100,6 +100,20 @@ fn test_rpc_send_tx() {
 
     assert_eq!(confirmed_tx, true);
 
+    use solana_account_decoder::UiAccountEncoding;
+    use solana_client::rpc_config::RpcAccountInfoConfig;
+    let config = RpcAccountInfoConfig {
+        encoding: Some(UiAccountEncoding::Binary64),
+        commitment: None,
+        data_slice: None,
+    };
+    let req = json_req!(
+        "getAccountInfo",
+        json!([bs58::encode(bob_pubkey).into_string(), config])
+    );
+    let json: Value = post_rpc(req, &leader_data);
+    info!("{:?}", json["result"]["value"]);
+
     server.close().unwrap();
     remove_dir_all(ledger_path).unwrap();
 }
@@ -189,7 +203,7 @@ fn test_rpc_subscriptions() {
             .and_then(move |client| {
                 for sig in signature_set {
                     let status_sender = status_sender.clone();
-                    tokio::spawn(
+                    tokio_01::spawn(
                         client
                             .signature_subscribe(sig.clone(), None)
                             .and_then(move |sig_stream| {
@@ -203,7 +217,7 @@ fn test_rpc_subscriptions() {
                             }),
                     );
                 }
-                tokio::spawn(
+                tokio_01::spawn(
                     client
                         .slot_subscribe()
                         .and_then(move |slot_stream| {
@@ -218,7 +232,7 @@ fn test_rpc_subscriptions() {
                 );
                 for pubkey in account_set {
                     let account_sender = account_sender.clone();
-                    tokio::spawn(
+                    tokio_01::spawn(
                         client
                             .account_subscribe(pubkey, None)
                             .and_then(move |account_stream| {

crate-features/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-crate-features"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana Crate Features"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

docs/src/apps/jsonrpc-api.md

@@ -46,10 +46,6 @@ To interact with a Solana node inside a JavaScript application, use the [solana-
 - [getSlotLeader](jsonrpc-api.md#getslotleader)
 - [getStakeActivation](jsonrpc-api.md#getstakeactivation)
 - [getSupply](jsonrpc-api.md#getsupply)
-- [getTokenAccountBalance](jsonrpc-api.md#gettokenaccountbalance)
-- [getTokenAccountsByDelegate](jsonrpc-api.md#gettokenaccountsbydelegate)
-- [getTokenAccountsByOwner](jsonrpc-api.md#gettokenaccountsbyowner)
-- [getTokenSupply](jsonrpc-api.md#gettokensupply)
 - [getTransactionCount](jsonrpc-api.md#gettransactioncount)
 - [getVersion](jsonrpc-api.md#getversion)
 - [getVoteAccounts](jsonrpc-api.md#getvoteaccounts)
@@ -69,6 +65,16 @@ To interact with a Solana node inside a JavaScript application, use the [solana-
   - [slotSubscribe](jsonrpc-api.md#slotsubscribe)
   - [slotUnsubscribe](jsonrpc-api.md#slotunsubscribe)
 
+## Unstable Methods
+
+Unstable methods may see breaking changes in patch releases and may not be supported in perpetuity.
+
+- [getTokenAccountBalance](jsonrpc-api.md#gettokenaccountbalance)
+- [getTokenAccountsByDelegate](jsonrpc-api.md#gettokenaccountsbydelegate)
+- [getTokenAccountsByOwner](jsonrpc-api.md#gettokenaccountsbyowner)
+- [getTokenLargestAccounts](jsonrpc-api.md#gettokenlargestaccounts)
+- [getTokenSupply](jsonrpc-api.md#gettokensupply)
+
 ## Request Formatting
 
 To make a JSON-RPC request, send an HTTP POST request with a `Content-Type: application/json` header. The JSON request data should contain 4 fields:
@@ -151,8 +157,9 @@ Returns all information associated with the account of provided Pubkey
 - `<string>` - Pubkey of account to query, as base-58 encoded string
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-  - (optional) `encoding: <string>` - encoding for Account data, either "binary" or jsonParsed". If parameter not provided, the default encoding is binary.
-    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`.
+  - (optional) `encoding: <string>` - encoding for Account data, either "binary", "binary64", or jsonParsed". If parameter not provided, the default encoding is "binary". "binary" is base-58 encoded and limited to Account data of less than 128 bytes. "binary64" will return base64 encoded data for Account data of any size.
+    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`. **jsonParsed encoding is UNSTABLE**
+  - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "binary" or "binary64" encoding.
 
 #### Results:
 
@@ -300,7 +307,7 @@ Returns identity and transaction information about a confirmed block in the ledg
 #### Parameters:
 
 - `<u64>` - slot, as u64 integer
-- `<string>` - (optional) encoding for each returned Transaction, either "json", "jsonParsed", or "binary". If parameter not provided, the default encoding is JSON.
+- `<string>` - (optional) encoding for each returned Transaction, either "json", "jsonParsed", or "binary". If parameter not provided, the default encoding is JSON. **jsonParsed encoding is UNSTABLE**
   Parsed-JSON encoding attempts to use program-specific instruction parsers to return more human-readable and explicit data in the `transaction.message.instructions` list. If parsed-JSON is requested but a parser cannot be found, the instruction falls back to regular JSON encoding (`accounts`, `data`, and `programIdIndex` fields).
 
 #### Results:
@@ -458,7 +465,7 @@ Returns transaction details for a confirmed transaction
 
 - `<string>` - transaction signature as base-58 encoded string
 N encoding attempts to use program-specific instruction parsers to return more human-readable and explicit data in the `transaction.message.instructions` list. If parsed-JSON is requested but a parser cannot be found, the instruction falls back to regular JSON encoding (`accounts`, `data`, and `programIdIndex` fields).
-- `<string>` - (optional) encoding for the returned Transaction, either "json", "jsonParsed", or "binary".
+- `<string>` - (optional) encoding for the returned Transaction, either "json", "jsonParsed", or "binary". **jsonParsed encoding is UNSTABLE**
 
 #### Results:
 
@@ -839,7 +846,8 @@ Returns all accounts owned by the provided program Pubkey
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
   - (optional) `encoding: <string>` - encoding for Account data, either "binary" or jsonParsed". If parameter not provided, the default encoding is binary.
-    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`.
+    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`. **jsonParsed encoding is UNSTABLE**
+  - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "binary" or "binary64" encoding.
   - (optional) `filters: <array>` - filter results using various [filter objects](jsonrpc-api.md#filters); account must meet all filter criteria to be included in results
 
 ##### Filters:
@@ -1056,7 +1064,7 @@ curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "
 
 ### getTokenAccountBalance
 
-Returns the token balance of an SPL Token account.
+Returns the token balance of an SPL Token account. **UNSTABLE**
 
 #### Parameters:
 
@@ -1082,7 +1090,7 @@ curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "
 
 ### getTokenAccountsByDelegate
 
-Returns all SPL Token accounts by approved Delegate.
+Returns all SPL Token accounts by approved Delegate. **UNSTABLE**
 
 #### Parameters:
 
@@ -1093,7 +1101,8 @@ Returns all SPL Token accounts by approved Delegate.
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
   - (optional) `encoding: <string>` - encoding for Account data, either "binary" or jsonParsed". If parameter not provided, the default encoding is binary.
-    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`.
+    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`. **jsonParsed encoding is UNSTABLE**
+  - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "binary" or "binary64" encoding.
 
 #### Results:
 
@@ -1113,12 +1122,12 @@ The result will be an RpcResponse JSON object with `value` equal to an array of
 // Request
 curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getTokenAccountsByDelegate", "params": ["4Nd1mBQtrMJVYVfKf2PJy9NZUZdTAsp7D4xWLs4gDB4T", {"programId": "TokenSVp5gheXUvJ6jGWGeCsgPKgnE3YgdGKRVCMY9o"}, {"encoding": "jsonParsed"}]}' http://localhost:8899
 // Result
-{"jsonrpc":"2.0","result":{"context":{"slot":1114},"value":[{"data":{"program":"spl-token","parsed":{"accountType":"account","info":{"amount":1,"delegate":"4Nd1mBQtrMJVYVfKf2PJy9NZUZdTAsp7D4xWLs4gDB4T","delegatedAmount":1,"isInitialized":true,"isNative":false,"mint":"3wyAj7Rt1TWVPZVteFJPLa26JmLvdb1CAKEFZm3NY75E","owner":"CnPoSPKXu7wJqxe59Fs72tkBeALovhsCxYeFwPCQH9TD"}}},"executable":false,"lamports":1726080,"owner":"TokenSVp5gheXUvJ6jGWGeCsgPKgnE3YgdGKRVCMY9o","rentEpoch":4},"pubkey":"CnPoSPKXu7wJqxe59Fs72tkBeALovhsCxYeFwPCQH9TD"}],"id":1}
+{"jsonrpc":"2.0","result":{"context":{"slot":1114},"value":[{"data":{"program":"spl-token","parsed":{"accountType":"account","info":{"tokenAmount":{"amount":"1","uiAmount":0.1,"decimals":1},"delegate":"4Nd1mBQtrMJVYVfKf2PJy9NZUZdTAsp7D4xWLs4gDB4T","delegatedAmount":1,"isInitialized":true,"isNative":false,"mint":"3wyAj7Rt1TWVPZVteFJPLa26JmLvdb1CAKEFZm3NY75E","owner":"CnPoSPKXu7wJqxe59Fs72tkBeALovhsCxYeFwPCQH9TD"}}},"executable":false,"lamports":1726080,"owner":"TokenSVp5gheXUvJ6jGWGeCsgPKgnE3YgdGKRVCMY9o","rentEpoch":4},"pubkey":"CnPoSPKXu7wJqxe59Fs72tkBeALovhsCxYeFwPCQH9TD"}],"id":1}
 ```
 
 ### getTokenAccountsByOwner
 
-Returns all SPL Token accounts by token owner.
+Returns all SPL Token accounts by token owner. **UNSTABLE**
 
 #### Parameters:
 
@@ -1129,7 +1138,8 @@ Returns all SPL Token accounts by token owner.
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
   - (optional) `encoding: <string>` - encoding for Account data, either "binary" or jsonParsed". If parameter not provided, the default encoding is binary.
-    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`.
+    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`. **jsonParsed encoding is UNSTABLE**
+  - (optional) `dataSlice: <object>` - limit the returned account data using the provided `offset: <usize>` and `length: <usize>` fields; only available for "binary" or "binary64" encoding.
 
 #### Results:
 
@@ -1149,12 +1159,39 @@ The result will be an RpcResponse JSON object with `value` equal to an array of
 // Request
 curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getTokenAccountsByOwner", "params": ["4Qkev8aNZcqFNSRhQzwyLMFSsi94jHqE8WNVTJzTP99F", {"mint":"3wyAj7Rt1TWVPZVteFJPLa26JmLvdb1CAKEFZm3NY75E"}, {"encoding": "jsonParsed"}]}' http://localhost:8899
 // Result
-{"jsonrpc":"2.0","result":{"context":{"slot":1114},"value":[{"data":{"program":"spl-token","parsed":{"accountType":"account","info":{"amount":1,"delegate":null,"delegatedAmount":1,"isInitialized":true,"isNative":false,"mint":"3wyAj7Rt1TWVPZVteFJPLa26JmLvdb1CAKEFZm3NY75E","owner":"4Qkev8aNZcqFNSRhQzwyLMFSsi94jHqE8WNVTJzTP99F"}}},"executable":false,"lamports":1726080,"owner":"TokenSVp5gheXUvJ6jGWGeCsgPKgnE3YgdGKRVCMY9o","rentEpoch":4},"pubkey":"CnPoSPKXu7wJqxe59Fs72tkBeALovhsCxYeFwPCQH9TD"}],"id":1}
+{"jsonrpc":"2.0","result":{"context":{"slot":1114},"value":[{"data":{"program":"spl-token","parsed":{"accountType":"account","info":{"tokenAmount":{"amount":"1","uiAmount":0.1,"decimals":1},"delegate":null,"delegatedAmount":1,"isInitialized":true,"isNative":false,"mint":"3wyAj7Rt1TWVPZVteFJPLa26JmLvdb1CAKEFZm3NY75E","owner":"4Qkev8aNZcqFNSRhQzwyLMFSsi94jHqE8WNVTJzTP99F"}}},"executable":false,"lamports":1726080,"owner":"TokenSVp5gheXUvJ6jGWGeCsgPKgnE3YgdGKRVCMY9o","rentEpoch":4},"pubkey":"CnPoSPKXu7wJqxe59Fs72tkBeALovhsCxYeFwPCQH9TD"}],"id":1}
+```
+
+### getTokenLargestAccounts
+
+Returns the 20 largest accounts of a particular SPL Token type. **UNSTABLE**
+
+#### Parameters:
+
+- `<string>` - Pubkey of token Mint to query, as base-58 encoded string
+- `<object>` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
+
+#### Results:
+
+The result will be an RpcResponse JSON object with `value` equal to an array of JSON objects containing:
+
+- `address: <string>` - the address of the token account
+- `uiAmount: <f64>` - the token account balance, using mint-prescribed decimals
+- `amount: <string>` - the raw token account balance without decimals, a string representation of u64
+- `decimals: <u8>` - number of base 10 digits to the right of the decimal place
+
+#### Example:
+
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getTokenLargestAccounts", "params": ["3wyAj7Rt1TWVPZVteFJPLa26JmLvdb1CAKEFZm3NY75E"]}' http://localhost:8899
+// Result
+{"jsonrpc":"2.0","result":{"context":{"slot":1114},"value":[{"address":"FYjHNoFtSQ5uijKrZFyYAxvEr87hsKXkXcxkcmkBAf4r","amount":"771","decimals":2,"uiAmount":7.71},{"address":"BnsywxTcaYeNUtzrPxQUvzAWxfzZe3ZLUJ4wMMuLESnu","amount":"229","decimals":2,"uiAmount":2.29}],"id":1}
 ```
 
 ### getTokenSupply
 
-Returns the total supply of an SPL Token type.
+Returns the total supply of an SPL Token type. **UNSTABLE**
 
 #### Parameters:
 
@@ -1220,7 +1257,7 @@ The result field will be a JSON object with the following fields:
 // Request
 curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getVersion"}' http://localhost:8899
 // Result
-{"jsonrpc":"2.0","result":{"solana-core": "1.2.19"},"id":1}
+{"jsonrpc":"2.0","result":{"solana-core": "1.2.24"},"id":1}
 ```
 
 ### getVoteAccounts
@@ -1346,7 +1383,7 @@ Simulate sending a transaction
 An RpcResponse containing a TransactionStatus object
 The result will be an RpcResponse JSON object with `value` set to a JSON object with the following fields:
 
-- `err: <object | null>` - Error if transaction failed, null if transaction succeeded. [TransactionError definitions](https://github.com/solana-labs/solana/blob/master/sdk/src/transaction.rs#L14)
+- `err: <object | string | null>` - Error if transaction failed, null if transaction succeeded. [TransactionError definitions](https://github.com/solana-labs/solana/blob/master/sdk/src/transaction.rs#L14)
 - `logs: <array | null>` - Array of log messages the transaction instructions output during execution, null if simulation failed before the transaction was able to execute (for example due to an invalid blockhash or signature verification failure)
 
 #### Example:
@@ -1356,7 +1393,7 @@ The result will be an RpcResponse JSON object with `value` set to a JSON object
 curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"simulateTransaction", "params":["4hXTCkRzt9WyecNzV1XPgCDfGAZzQKNxLXgynz5QDuWWPSAZBZSHptvWRL3BjCvzUXRdKvHL2b7yGrRQcWyaqsaBCncVG7BFggS8w9snUts67BSh3EqKpXLUm5UMHfD7ZBe9GhARjbNQMLJ1QD3Spr6oMTBU6EhdB4RD8CP2xUxr2u3d6fos36PD98XS6oX8TQjLpsMwncs5DAMiD4nNnR8NBfyghGCWvCVifVwvA8B8TJxE1aiyiv2L429BCWfyzAme5sZW8rDb14NeCQHhZbtNqfXhcp2tAnaAT"]}' http://localhost:8899
 
 // Result
-{"jsonrpc":"2.0","result":{"context":{"slot":218},"value":{"confirmations":0,"err":null,"slot":218,"status":{"Ok":null}}},"id":1}
+{"jsonrpc":"2.0","result":{"context":{"slot":218},"value":{"err":null,"logs":["BPF program 83astBRguLMdt2h5U1Tpdq5tjFoJ6noeGwaY3mDLVcri success"]},"id":1}
 ```
 
 ### setLogFilter
@@ -1421,7 +1458,7 @@ Subscribe to an account to receive notifications when the lamports or data for a
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - `<object>` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
   - (optional) `encoding: <string>` - encoding for Account data, either "binary" or jsonParsed". If parameter not provided, the default encoding is binary.
-    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`.
+    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`. **jsonParsed encoding is UNSTABLE**
 
 #### Results:
 
@@ -1476,8 +1513,10 @@ Subscribe to an account to receive notifications when the lamports or data for a
       },
       "value": {
         "data": {
-           "nonce": {
-              "initialized": {
+           "program": "nonce"
+           "parsed": {
+              "type": "initialized",
+              "info": {
                  "authority": "Bbqg1M4YVVfbhEzwA9SpC9FhsaG83YMTYoR4a8oTDLX",
                  "blockhash": "LUaQTmM7WbMRiATdMMHaRGakPtCkc2GHtH57STKXs6k",
                  "feeCalculator": {
@@ -1529,7 +1568,7 @@ Subscribe to a program to receive notifications when the lamports or data for a
 - `<object>` - (optional) Configuration object containing the following optional fields:
   - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
   - (optional) `encoding: <string>` - encoding for Account data, either "binary" or jsonParsed". If parameter not provided, the default encoding is binary.
-    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`.
+    Parsed-JSON encoding attempts to use program-specific state parsers to return more human-readable and explicit account state data. If parsed-JSON is requested but a parser cannot be found, the field falls back to binary encoding, detectable when the `data` field is type `<string>`. **jsonParsed encoding is UNSTABLE**
   - (optional) `filters: <array>` - filter results using various [filter objects](jsonrpc-api.md#filters); account must meet all filter criteria to be included in results
 
 #### Results:
@@ -1592,8 +1631,10 @@ Subscribe to a program to receive notifications when the lamports or data for a
         "pubkey": "H4vnBqifaSACnKa7acsxstsY1iV1bvJNxsCY7enrd1hq"
         "account": {
           "data": {
-             "nonce": {
-                "initialized": {
+             "program": "nonce"
+             "parsed": {
+                "type": "initialized",
+                "info": {
                    "authority": "Bbqg1M4YVVfbhEzwA9SpC9FhsaG83YMTYoR4a8oTDLX",
                    "blockhash": "LUaQTmM7WbMRiATdMMHaRGakPtCkc2GHtH57STKXs6k",
                    "feeCalculator": {

docs/src/implemented-proposals/rent.md

@@ -14,13 +14,21 @@ Currently, the rent cost is fixed at the genesis. However, it's anticipated to b
 
 There are two timings of collecting rent from accounts: \(1\) when referenced by a transaction, \(2\) periodically once an epoch. \(1\) includes the transaction to create the new account itself, and it happens during the normal transaction processing by the bank as part of the load phase. \(2\) exists to ensure to collect rents from stale accounts, which aren't referenced in recent epochs at all. \(2\) requires the whole scan of accounts and is spread over an epoch based on account address prefix to avoid load spikes due to this rent collection.
 
+On the contrary, rent collection isn't applied to accounts that are directly manipulated by any of protocol-level bookkeeping processes including:
+
+- The distribution of rent collection itself (Otherwise, it may cause recursive rent collection handling)
+- The distribution of staking rewards at the start of every epoch (To reduce as much as processing spike at the start of new epoch)
+- The distribution of transaction fee at the end of every slot
+
+Even if those processes are out of scope of rent collection, all of manipulated accounts will eventually be handled by the \(2\) mechanism.
+
 ## Actual processing of collecting rent
 
-Rent is due for one epoch's worth of time, and accounts always have `Account::rent_epoch` of `current_epoch + 1`.
+Rent is due for one epoch's worth of time, and accounts have `Account::rent_epoch` of `current_epoch` or `current_epoch + 1` depending on the rent regime.
 
-If the account is in the exempt regime, `Account::rent_epoch` is simply pushed to `current_epoch + 1`.
+If the account is in the exempt regime, `Account::rent_epoch` is simply updated to `current_epoch`.
 
-If the account is non-exempt, the difference between the next epoch and `Account::rent_epoch` is used to calculate the amount of rent owed by this account \(via `Rent::due()`\). Any fractional lamports of the calculation are truncated. Rent due is deducted from `Account::lamports` and `Account::rent_epoch` is updated to the next epoch. If the amount of rent due is less than one lamport, no changes are made to the account.
+If the account is non-exempt, the difference between the next epoch and `Account::rent_epoch` is used to calculate the amount of rent owed by this account \(via `Rent::due()`\). Any fractional lamports of the calculation are truncated. Rent due is deducted from `Account::lamports` and `Account::rent_epoch` is updated to `current_epoch + 1` (= next epoch). If the amount of rent due is less than one lamport, no changes are made to the account.
 
 Accounts whose balance is insufficient to satisfy the rent that would be due simply fail to load.
 

docs/src/implemented-proposals/rpc-transaction-history.md

@@ -0,0 +1,106 @@
+# Long term RPC Transaction History
+There's a need for RPC to serve at least 6 months of transaction history.  The
+current history, on the order of days, is insufficient for downstream users.
+
+6 months of transaction data cannot be stored practically in a validator's
+rocksdb ledger so an external data store is necessary.   The validator's
+rocksdb ledger will continue to serve as the primary data source, and then will
+fall back to the external data store.
+
+The affected RPC endpoints are:
+* [getFirstAvailableBlock](https://docs.solana.com/apps/jsonrpc-api#getfirstavailableblock)
+* [getConfirmedBlock](https://docs.solana.com/apps/jsonrpc-api#getconfirmedblock)
+* [getConfirmedBlocks](https://docs.solana.com/apps/jsonrpc-api#getconfirmedblocks)
+* [getConfirmedSignaturesForAddress](https://docs.solana.com/apps/jsonrpc-api#getconfirmedsignaturesforaddress)
+* [getConfirmedTransaction](https://docs.solana.com/apps/jsonrpc-api#getconfirmedtransaction)
+* [getSignatureStatuses](https://docs.solana.com/apps/jsonrpc-api#getsignaturestatuses)
+
+Note that [getBlockTime](https://docs.solana.com/apps/jsonrpc-api#getblocktime)
+is not supported, as once https://github.com/solana-labs/solana/issues/10089 is
+fixed then `getBlockTime` can be removed.
+
+Some system design constraints:
+* The volume of data to store and search can quickly jump into the terabytes,
+  and is immutable.
+* The system should be as light as possible for SREs.  For example an SQL
+  database cluster that requires an SRE to continually monitor and rebalance
+  nodes is undesirable.
+* Data must be searchable in real time - batched queries that take minutes or
+  hours to run are unacceptable.
+* Easy to replicate the data worldwide to co-locate it with the RPC endpoints
+  that will utilize it.
+* Interfacing with the external data store should be easy and not require
+  depending on risky lightly-used community-supported code libraries
+
+Based on these constraints, Google's BigTable product is selected as the data
+store.
+
+## Table Schema
+A BigTable instance is used to hold all transaction data, broken up into
+different tables for quick searching.
+
+New data may be copied into the instance at anytime without affecting the existing
+data, and all data is immutable.  Generally the expectation is that new data
+will be uploaded once an current epoch completes but there is no limitation on
+the frequency of data dumps.
+
+Cleanup of old data is automatic by configuring the data retention policy of the
+instance tables appropriately, it just disappears.  Therefore the order of when data is
+added becomes important.  For example if data from epoch N-1 is added after data
+from epoch N, the older epoch data will outlive the newer data.  However beyond
+producing _holes_ in query results, this kind of unordered deletion will
+have no ill effect.  Note that this method of cleanup effectively allows for an
+unlimited amount of transaction data to be stored, restricted only by the
+monetary costs of doing so.
+
+The table layout s supports the existing RPC endpoints only.  New RPC endpoints
+in the future may require additions to the schema and potentially iterating over
+all transactions to build up the necessary metadata.
+
+## Accessing BigTable
+BigTable has a gRPC endpoint that can be accessed using the
+[tonic](https://crates.io/crates/crate)] and the raw protobuf API, as currently no
+higher-level Rust crate for BigTable exists.  Practically this makes parsing the
+results of BigTable queries more complicated but is not a significant issue.
+
+## Data Population
+The ongoing population of instance data will occur on an epoch cadence through the
+use of a new `solana-ledger-tool` command that will convert rocksdb data for a
+given slot range into the instance schema.
+
+The same process will be run once, manually, to backfill the existing ledger
+data.
+
+### Block Table: `block`
+
+This table contains the compressed block data for a given slot.
+
+The row key is generated by taking the 16 digit lower case hexadecimal
+representation of the slot, to ensure that the oldest slot with a confirmed
+block will always be first when the rows are listed.  eg, The row key for slot
+42 would be 000000000000002a.
+
+The row data is a compressed `StoredConfirmedBlock` struct.
+
+
+### Account Address Transaction Signature Lookup Table: `tx-by-addr`
+
+This table contains the transactions that affect a given address.
+
+The row key is `<base58
+address>/<slot-id-one's-compliment-hex-slot-0-prefixed-to-16-digits>`.  The row
+data is a compressed `TransactionByAddrInfo` struct.
+
+Taking the one's compliment of the slot allows for listing of slots ensures that
+the newest slot with transactions that affect an address will always
+be listed first.
+
+Sysvar addresses are not indexed.  However frequently used programs such as
+Vote or System are, and will likely have a row for every confirmed slot.
+
+### Transaction Signature Lookup Table: `tx`
+
+This table maps a transaction signature to its confirmed block, and index within that block.
+
+The row key is the base58-encoded transaction signature.
+The row data is a compressed `TransactionInfo` struct.

dos/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-dos"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,14 +13,14 @@ clap = "2.33.1"
 log = "0.4.8"
 rand = "0.7.0"
 rayon = "1.3.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-core = { path = "../core", version = "1.2.19" }
-solana-ledger = { path = "../ledger", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-net-utils = { path = "../net-utils", version = "1.2.19" }
-solana-runtime = { path = "../runtime", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-core = { path = "../core", version = "1.2.24" }
+solana-ledger = { path = "../ledger", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-net-utils = { path = "../net-utils", version = "1.2.24" }
+solana-runtime = { path = "../runtime", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

download-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-download-utils"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana Download Utils"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,8 +14,8 @@ console = "0.10.1"
 indicatif = "0.14.0"
 log = "0.4.8"
 reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-ledger = { path = "../ledger", version = "1.2.19" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-ledger = { path = "../ledger", version = "1.2.24" }
 tar = "0.4.28"
 
 [lib]

faucet/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-faucet"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana Faucet"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -16,11 +16,11 @@ clap = "2.33"
 log = "0.4.8"
 serde = "1.0.110"
 serde_derive = "1.0.103"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-metrics = { path = "../metrics", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-metrics = { path = "../metrics", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
 tokio = "0.1"
 tokio-codec = "0.1"
 

faucet/src/bin/faucet.rs

@@ -1,4 +1,5 @@
 use clap::{crate_description, crate_name, App, Arg};
+use solana_clap_utils::input_parsers::{lamports_of_sol, value_of};
 use solana_faucet::{
     faucet::{run_faucet, Faucet, FAUCET_PORT},
     socketaddr,
@@ -34,36 +35,36 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                 .help("Time slice over which to limit requests to faucet"),
         )
         .arg(
-            Arg::with_name("cap")
-                .long("cap")
+            Arg::with_name("per_time_cap")
+                .long("per-time-cap")
+                .alias("cap")
                 .value_name("NUM")
                 .takes_value(true)
-                .help("Request limit for time slice"),
+                .help("Request limit for time slice, in SOL"),
+        )
+        .arg(
+            Arg::with_name("per_request_cap")
+                .long("per-request-cap")
+                .value_name("NUM")
+                .takes_value(true)
+                .help("Request limit for a single request, in SOL"),
         )
         .get_matches();
 
     let mint_keypair = read_keypair_file(matches.value_of("keypair").unwrap())
         .expect("failed to read client keypair");
 
-    let time_slice: Option<u64>;
-    if let Some(secs) = matches.value_of("slice") {
-        time_slice = Some(secs.to_string().parse().expect("failed to parse slice"));
-    } else {
-        time_slice = None;
-    }
-    let request_cap: Option<u64>;
-    if let Some(c) = matches.value_of("cap") {
-        request_cap = Some(c.to_string().parse().expect("failed to parse cap"));
-    } else {
-        request_cap = None;
-    }
+    let time_slice = value_of(&matches, "slice");
+    let per_time_cap = lamports_of_sol(&matches, "per_time_cap");
+    let per_request_cap = lamports_of_sol(&matches, "per_request_cap");
 
     let faucet_addr = socketaddr!(0, FAUCET_PORT);
 
     let faucet = Arc::new(Mutex::new(Faucet::new(
         mint_keypair,
         time_slice,
-        request_cap,
+        per_time_cap,
+        per_request_cap,
     )));
 
     let faucet1 = faucet.clone();

faucet/src/faucet.rs

@@ -62,7 +62,8 @@ pub struct Faucet {
     mint_keypair: Keypair,
     ip_cache: Vec<IpAddr>,
     pub time_slice: Duration,
-    request_cap: u64,
+    per_time_cap: u64,
+    per_request_cap: Option<u64>,
     pub request_current: u64,
 }
 
@@ -70,27 +71,23 @@ impl Faucet {
     pub fn new(
         mint_keypair: Keypair,
         time_input: Option<u64>,
-        request_cap_input: Option<u64>,
+        per_time_cap: Option<u64>,
+        per_request_cap: Option<u64>,
     ) -> Faucet {
-        let time_slice = match time_input {
-            Some(time) => Duration::new(time, 0),
-            None => Duration::new(TIME_SLICE, 0),
-        };
-        let request_cap = match request_cap_input {
-            Some(cap) => cap,
-            None => REQUEST_CAP,
-        };
+        let time_slice = Duration::new(time_input.unwrap_or(TIME_SLICE), 0);
+        let per_time_cap = per_time_cap.unwrap_or(REQUEST_CAP);
         Faucet {
             mint_keypair,
             ip_cache: Vec::new(),
             time_slice,
-            request_cap,
+            per_time_cap,
+            per_request_cap,
             request_current: 0,
         }
     }
 
-    pub fn check_request_limit(&mut self, request_amount: u64) -> bool {
-        (self.request_current + request_amount) <= self.request_cap
+    pub fn check_time_request_limit(&mut self, request_amount: u64) -> bool {
+        (self.request_current + request_amount) <= self.per_time_cap
     }
 
     pub fn clear_request_count(&mut self) {
@@ -116,7 +113,15 @@ impl Faucet {
                 to,
                 blockhash,
             } => {
-                if self.check_request_limit(lamports) {
+                if let Some(cap) = self.per_request_cap {
+                    if lamports > cap {
+                        return Err(Error::new(
+                            ErrorKind::Other,
+                            format!("request too large; req: {} cap: {}", lamports, cap),
+                        ));
+                    }
+                }
+                if self.check_time_request_limit(lamports) {
                     self.request_current += lamports;
                     datapoint_info!(
                         "faucet-airdrop",
@@ -135,7 +140,7 @@ impl Faucet {
                         ErrorKind::Other,
                         format!(
                             "token limit reached; req: {} current: {} cap: {}",
-                            lamports, self.request_current, self.request_cap
+                            lamports, self.request_current, self.per_time_cap
                         ),
                     ))
                 }
@@ -248,14 +253,15 @@ pub fn request_airdrop_transaction(
 pub fn run_local_faucet(
     mint_keypair: Keypair,
     sender: Sender<SocketAddr>,
-    request_cap_input: Option<u64>,
+    per_time_cap: Option<u64>,
 ) {
     thread::spawn(move || {
         let faucet_addr = socketaddr!(0, 0);
         let faucet = Arc::new(Mutex::new(Faucet::new(
             mint_keypair,
             None,
-            request_cap_input,
+            per_time_cap,
+            None,
         )));
         run_faucet(faucet, faucet_addr, Some(sender));
     });
@@ -312,18 +318,18 @@ mod tests {
     use std::time::Duration;
 
     #[test]
-    fn test_check_request_limit() {
+    fn test_check_time_request_limit() {
         let keypair = Keypair::new();
-        let mut faucet = Faucet::new(keypair, None, Some(3));
-        assert!(faucet.check_request_limit(1));
+        let mut faucet = Faucet::new(keypair, None, Some(3), None);
+        assert!(faucet.check_time_request_limit(1));
         faucet.request_current = 3;
-        assert!(!faucet.check_request_limit(1));
+        assert!(!faucet.check_time_request_limit(1));
     }
 
     #[test]
     fn test_clear_request_count() {
         let keypair = Keypair::new();
-        let mut faucet = Faucet::new(keypair, None, None);
+        let mut faucet = Faucet::new(keypair, None, None, None);
         faucet.request_current += 256;
         assert_eq!(faucet.request_current, 256);
         faucet.clear_request_count();
@@ -333,7 +339,7 @@ mod tests {
     #[test]
     fn test_add_ip_to_cache() {
         let keypair = Keypair::new();
-        let mut faucet = Faucet::new(keypair, None, None);
+        let mut faucet = Faucet::new(keypair, None, None, None);
         let ip = "127.0.0.1".parse().expect("create IpAddr from string");
         assert_eq!(faucet.ip_cache.len(), 0);
         faucet.add_ip_to_cache(ip);
@@ -344,7 +350,7 @@ mod tests {
     #[test]
     fn test_clear_ip_cache() {
         let keypair = Keypair::new();
-        let mut faucet = Faucet::new(keypair, None, None);
+        let mut faucet = Faucet::new(keypair, None, None, None);
         let ip = "127.0.0.1".parse().expect("create IpAddr from string");
         assert_eq!(faucet.ip_cache.len(), 0);
         faucet.add_ip_to_cache(ip);
@@ -359,9 +365,10 @@ mod tests {
         let keypair = Keypair::new();
         let time_slice: Option<u64> = None;
         let request_cap: Option<u64> = None;
-        let faucet = Faucet::new(keypair, time_slice, request_cap);
+        let faucet = Faucet::new(keypair, time_slice, request_cap, Some(100));
         assert_eq!(faucet.time_slice, Duration::new(TIME_SLICE, 0));
-        assert_eq!(faucet.request_cap, REQUEST_CAP);
+        assert_eq!(faucet.per_time_cap, REQUEST_CAP);
+        assert_eq!(faucet.per_request_cap, Some(100));
     }
 
     #[test]
@@ -376,7 +383,7 @@ mod tests {
 
         let mint = Keypair::new();
         let mint_pubkey = mint.pubkey();
-        let mut faucet = Faucet::new(mint, None, None);
+        let mut faucet = Faucet::new(mint, None, None, None);
 
         let tx = faucet.build_airdrop_transaction(request).unwrap();
         let message = tx.message();
@@ -392,8 +399,15 @@ mod tests {
         let instruction: SystemInstruction = deserialize(&message.instructions[0].data).unwrap();
         assert_eq!(instruction, SystemInstruction::Transfer { lamports: 2 });
 
+        // Test per-time request cap
         let mint = Keypair::new();
-        faucet = Faucet::new(mint, None, Some(1));
+        faucet = Faucet::new(mint, None, Some(1), None);
+        let tx = faucet.build_airdrop_transaction(request);
+        assert!(tx.is_err());
+
+        // Test per-request cap
+        let mint = Keypair::new();
+        faucet = Faucet::new(mint, None, None, Some(1));
         let tx = faucet.build_airdrop_transaction(request);
         assert!(tx.is_err());
     }
@@ -421,7 +435,7 @@ mod tests {
         LittleEndian::write_u16(&mut expected_vec_with_length, expected_bytes.len() as u16);
         expected_vec_with_length.extend_from_slice(&expected_bytes);
 
-        let mut faucet = Faucet::new(keypair, None, None);
+        let mut faucet = Faucet::new(keypair, None, None, None);
         let response = faucet.process_faucet_request(&bytes);
         let response_vec = response.unwrap().to_vec();
         assert_eq!(expected_vec_with_length, response_vec);

genesis-programs/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-genesis-programs"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana genesis programs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,12 +10,12 @@ edition = "2018"
 
 [dependencies]
 log = { version = "0.4.8" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.2.19" }
-solana-budget-program = { path = "../programs/budget", version = "1.2.19" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.2.19" }
-solana-runtime = { path = "../runtime", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-vest-program = { path = "../programs/vest", version = "1.2.19" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.2.24" }
+solana-budget-program = { path = "../programs/budget", version = "1.2.24" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.2.24" }
+solana-runtime = { path = "../runtime", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-vest-program = { path = "../programs/vest", version = "1.2.24" }
 
 [lib]
 crate-type = ["lib"]

genesis-programs/src/lib.rs

@@ -105,7 +105,11 @@ pub fn get_entered_epoch_callback(operating_mode: OperatingMode) -> EnteredEpoch
                 bank.add_native_program(name, program_id);
             }
         }
-        bank.set_cross_program_support(OperatingMode::Stable != operating_mode);
+        if OperatingMode::Stable == operating_mode {
+            bank.set_cross_program_support(bank.epoch() >= 63);
+        } else {
+            bank.set_cross_program_support(true);
+        }
     })
 }
 

genesis/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-genesis"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,14 +15,14 @@ chrono = "0.4"
 serde = "1.0.110"
 serde_json = "1.0.53"
 serde_yaml = "0.8.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-genesis-programs = { path = "../genesis-programs", version = "1.2.19" }
-solana-ledger = { path = "../ledger", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-stake-program = { path = "../programs/stake", version = "1.2.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.2.24" }
+solana-ledger = { path = "../ledger", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.24" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
 tempfile = "3.1.0"
 
 [[bin]]

gossip/Cargo.toml

@@ -3,20 +3,20 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-gossip"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-core = { path = "../core", version = "1.2.19" }
-solana-client = { path = "../client", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-net-utils = { path = "../net-utils", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-core = { path = "../core", version = "1.2.24" }
+solana-client = { path = "../client", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-net-utils = { path = "../net-utils", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

install/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-install"
 description = "The solana cluster software installer"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -24,12 +24,12 @@ reqwest = { version = "0.10.4", default-features = false, features = ["blocking"
 serde = "1.0.110"
 serde_derive = "1.0.103"
 serde_yaml = "0.8.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-client = { path = "../client", version = "1.2.19" }
-solana-config-program = { path = "../programs/config", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-client = { path = "../client", version = "1.2.24" }
+solana-config-program = { path = "../programs/config", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
 semver = "0.9.0"
 tar = "0.4.28"
 tempdir = "0.3.7"

keygen/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-keygen"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana key generation utility"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,11 +13,11 @@ bs58 = "0.3.1"
 clap = "2.33"
 dirs = "2.0.2"
 num_cpus = "1.13.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-cli-config = { path = "../cli-config", version = "1.2.19" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-cli-config = { path = "../cli-config", version = "1.2.24" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
 tiny-bip39 = "0.7.0"
 
 [[bin]]

ledger-tool/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-ledger-tool"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -12,22 +12,27 @@ homepage = "https://solana.com/"
 bs58 = "0.3.1"
 bytecount = "0.6.0"
 clap = "2.33.1"
+futures = "0.3.5"
+futures-util = "0.3.5"
 histogram = "*"
 log = { version = "0.4.8" }
+regex = "1"
 serde_json = "1.0.53"
 serde_yaml = "0.8.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-cli = { path = "../cli", version = "1.2.19" }
-solana-ledger = { path = "../ledger", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-runtime = { path = "../runtime", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-stake-program = { path = "../programs/stake", version = "1.2.19" }
-solana-transaction-status = { path = "../transaction-status", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-cli = { path = "../cli", version = "1.2.24" }
+solana-ledger = { path = "../ledger", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-measure = { path = "../measure", version = "1.2.24" }
+solana-runtime = { path = "../runtime", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.24" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "1.2.24" }
+solana-transaction-status = { path = "../transaction-status", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.24" }
 tempfile = "3.1.0"
-regex = "1"
+tokio = { version = "0.2.22", features = ["full"] }
 
 [dev-dependencies]
 assert_cmd = "1.0"

ledger-tool/src/bigtable.rs

@@ -0,0 +1,567 @@
+/// The `bigtable` subcommand
+use clap::{value_t, value_t_or_exit, App, AppSettings, Arg, ArgMatches, SubCommand};
+use log::*;
+use solana_clap_utils::{
+    input_parsers::pubkey_of,
+    input_validators::{is_slot, is_valid_pubkey},
+};
+use solana_cli::display::println_transaction;
+use solana_ledger::{blockstore::Blockstore, blockstore_db::AccessType};
+use solana_measure::measure::Measure;
+use solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature};
+use solana_transaction_status::UiTransactionEncoding;
+use std::{collections::HashSet, path::Path, process::exit, result::Result, time::Duration};
+use tokio::time::delay_for;
+
+// Attempt to upload this many blocks in parallel
+const NUM_BLOCKS_TO_UPLOAD_IN_PARALLEL: usize = 32;
+
+// Read up to this many blocks from blockstore before blocking on the upload process
+const BLOCK_READ_AHEAD_DEPTH: usize = NUM_BLOCKS_TO_UPLOAD_IN_PARALLEL * 2;
+
+async fn upload(
+    blockstore: Blockstore,
+    starting_slot: Slot,
+    ending_slot: Option<Slot>,
+    allow_missing_metadata: bool,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let mut measure = Measure::start("entire upload");
+
+    let bigtable = solana_storage_bigtable::LedgerStorage::new(false)
+        .await
+        .map_err(|err| format!("Failed to connect to storage: {:?}", err))?;
+
+    info!("Loading ledger slots...");
+    let blockstore_slots: Vec<_> = blockstore
+        .slot_meta_iterator(starting_slot)
+        .map_err(|err| {
+            format!(
+                "Failed to load entries starting from slot {}: {:?}",
+                starting_slot, err
+            )
+        })?
+        .filter_map(|(slot, _slot_meta)| {
+            if let Some(ending_slot) = &ending_slot {
+                if slot > *ending_slot {
+                    return None;
+                }
+            }
+            Some(slot)
+        })
+        .collect();
+
+    if blockstore_slots.is_empty() {
+        info!("Ledger has no slots in the specified range");
+        return Ok(());
+    }
+    info!(
+        "Found {} slots in the range ({}, {})",
+        blockstore_slots.len(),
+        blockstore_slots.first().unwrap(),
+        blockstore_slots.last().unwrap()
+    );
+
+    let mut blockstore_slots_with_no_confirmed_block = HashSet::new();
+
+    // Gather the blocks that are already present in bigtable, by slot
+    let bigtable_slots = {
+        let mut bigtable_slots = vec![];
+        let first_blockstore_slot = *blockstore_slots.first().unwrap();
+        let last_blockstore_slot = *blockstore_slots.last().unwrap();
+        info!(
+            "Loading list of bigtable blocks between slots {} and {}...",
+            first_blockstore_slot, last_blockstore_slot
+        );
+
+        let mut start_slot = *blockstore_slots.first().unwrap();
+        while start_slot <= last_blockstore_slot {
+            let mut next_bigtable_slots = loop {
+                match bigtable.get_confirmed_blocks(start_slot, 1000).await {
+                    Ok(slots) => break slots,
+                    Err(err) => {
+                        error!("get_confirmed_blocks for {} failed: {:?}", start_slot, err);
+                        // Consider exponential backoff...
+                        delay_for(Duration::from_secs(2)).await;
+                    }
+                }
+            };
+            if next_bigtable_slots.is_empty() {
+                break;
+            }
+            bigtable_slots.append(&mut next_bigtable_slots);
+            start_slot = bigtable_slots.last().unwrap() + 1;
+        }
+        bigtable_slots
+            .into_iter()
+            .filter(|slot| *slot <= last_blockstore_slot)
+            .collect::<Vec<_>>()
+    };
+
+    // The blocks that still need to be uploaded is the difference between what's already in the
+    // bigtable and what's in blockstore...
+    let blocks_to_upload = {
+        let blockstore_slots = blockstore_slots.iter().cloned().collect::<HashSet<_>>();
+        let bigtable_slots = bigtable_slots.into_iter().collect::<HashSet<_>>();
+
+        let mut blocks_to_upload = blockstore_slots
+            .difference(&blockstore_slots_with_no_confirmed_block)
+            .cloned()
+            .collect::<HashSet<_>>()
+            .difference(&bigtable_slots)
+            .cloned()
+            .collect::<Vec<_>>();
+        blocks_to_upload.sort();
+        blocks_to_upload
+    };
+
+    if blocks_to_upload.is_empty() {
+        info!("No blocks need to be uploaded to bigtable");
+        return Ok(());
+    }
+    info!(
+        "{} blocks to be uploaded to the bucket in the range ({}, {})",
+        blocks_to_upload.len(),
+        blocks_to_upload.first().unwrap(),
+        blocks_to_upload.last().unwrap()
+    );
+
+    // Load the blocks out of blockstore in a separate thread to allow for concurrent block uploading
+    let (_loader_thread, receiver) = {
+        let (sender, receiver) = std::sync::mpsc::sync_channel(BLOCK_READ_AHEAD_DEPTH);
+        (
+            std::thread::spawn(move || {
+                let mut measure = Measure::start("block loader thread");
+                for (i, slot) in blocks_to_upload.iter().enumerate() {
+                    let _ = match blockstore.get_confirmed_block(
+                        *slot,
+                        Some(solana_transaction_status::UiTransactionEncoding::Binary),
+                    ) {
+                        Ok(confirmed_block) => sender.send((*slot, Some(confirmed_block))),
+                        Err(err) => {
+                            warn!(
+                                "Failed to get load confirmed block from slot {}: {:?}",
+                                slot, err
+                            );
+                            sender.send((*slot, None))
+                        }
+                    };
+
+                    if i % NUM_BLOCKS_TO_UPLOAD_IN_PARALLEL == 0 {
+                        info!(
+                            "{}% of blocks processed ({}/{})",
+                            i * 100 / blocks_to_upload.len(),
+                            i,
+                            blocks_to_upload.len()
+                        );
+                    }
+                }
+                measure.stop();
+                info!("{} to load {} blocks", measure, blocks_to_upload.len());
+            }),
+            receiver,
+        )
+    };
+
+    let mut failures = 0;
+    use futures::stream::StreamExt;
+
+    let mut stream =
+        tokio::stream::iter(receiver.into_iter()).chunks(NUM_BLOCKS_TO_UPLOAD_IN_PARALLEL);
+
+    while let Some(blocks) = stream.next().await {
+        let mut measure_upload = Measure::start("Upload");
+        let mut num_blocks = blocks.len();
+        info!("Preparing the next {} blocks for upload", num_blocks);
+
+        let uploads = blocks.into_iter().filter_map(|(slot, block)| match block {
+            None => {
+                blockstore_slots_with_no_confirmed_block.insert(slot);
+                num_blocks -= 1;
+                None
+            }
+            Some(confirmed_block) => {
+                if confirmed_block
+                    .transactions
+                    .iter()
+                    .any(|transaction| transaction.meta.is_none())
+                {
+                    if allow_missing_metadata {
+                        info!("Transaction metadata missing from slot {}", slot);
+                    } else {
+                        panic!("Transaction metadata missing from slot {}", slot);
+                    }
+                }
+                Some(bigtable.upload_confirmed_block(slot, confirmed_block))
+            }
+        });
+
+        for result in futures::future::join_all(uploads).await {
+            if result.is_err() {
+                error!("upload_confirmed_block() failed: {:?}", result.err());
+                failures += 1;
+            }
+        }
+
+        measure_upload.stop();
+        info!("{} for {} blocks", measure_upload, num_blocks);
+    }
+
+    measure.stop();
+    info!("{}", measure);
+    if failures > 0 {
+        Err(format!("Incomplete upload, {} operations failed", failures).into())
+    } else {
+        Ok(())
+    }
+}
+
+async fn first_available_block() -> Result<(), Box<dyn std::error::Error>> {
+    let bigtable = solana_storage_bigtable::LedgerStorage::new(true).await?;
+    match bigtable.get_first_available_block().await? {
+        Some(block) => println!("{}", block),
+        None => println!("No blocks available"),
+    }
+
+    Ok(())
+}
+
+async fn block(slot: Slot) -> Result<(), Box<dyn std::error::Error>> {
+    let bigtable = solana_storage_bigtable::LedgerStorage::new(false)
+        .await
+        .map_err(|err| format!("Failed to connect to storage: {:?}", err))?;
+
+    let block = bigtable
+        .get_confirmed_block(slot, UiTransactionEncoding::Binary)
+        .await?;
+
+    println!("Slot: {}", slot);
+    println!("Parent Slot: {}", block.parent_slot);
+    println!("Blockhash: {}", block.blockhash);
+    println!("Previous Blockhash: {}", block.previous_blockhash);
+    if block.block_time.is_some() {
+        println!("Block Time: {:?}", block.block_time);
+    }
+    if !block.rewards.is_empty() {
+        println!("Rewards: {:?}", block.rewards);
+    }
+    for (index, transaction_with_meta) in block.transactions.iter().enumerate() {
+        println!("Transaction {}:", index);
+        println_transaction(
+            &transaction_with_meta.transaction.decode().unwrap(),
+            &transaction_with_meta.meta,
+            "  ",
+        );
+    }
+    Ok(())
+}
+
+async fn blocks(starting_slot: Slot, limit: usize) -> Result<(), Box<dyn std::error::Error>> {
+    let bigtable = solana_storage_bigtable::LedgerStorage::new(false)
+        .await
+        .map_err(|err| format!("Failed to connect to storage: {:?}", err))?;
+
+    let slots = bigtable.get_confirmed_blocks(starting_slot, limit).await?;
+    println!("{:?}", slots);
+    println!("{} blocks found", slots.len());
+
+    Ok(())
+}
+
+async fn confirm(signature: &Signature, verbose: bool) -> Result<(), Box<dyn std::error::Error>> {
+    let bigtable = solana_storage_bigtable::LedgerStorage::new(false)
+        .await
+        .map_err(|err| format!("Failed to connect to storage: {:?}", err))?;
+
+    let transaction_status = bigtable.get_signature_status(signature).await?;
+
+    if verbose {
+        match bigtable
+            .get_confirmed_transaction(signature, UiTransactionEncoding::Binary)
+            .await
+        {
+            Ok(Some(confirmed_transaction)) => {
+                println!(
+                    "\nTransaction executed in slot {}:",
+                    confirmed_transaction.slot
+                );
+                println_transaction(
+                    &confirmed_transaction
+                        .transaction
+                        .transaction
+                        .decode()
+                        .expect("Successful decode"),
+                    &confirmed_transaction.transaction.meta,
+                    "  ",
+                );
+            }
+            Ok(None) => println!("Confirmed transaction details not available"),
+            Err(err) => println!("Unable to get confirmed transaction details: {}", err),
+        }
+        println!();
+    }
+    match transaction_status.status {
+        Ok(_) => println!("Confirmed"),
+        Err(err) => println!("Transaction failed: {}", err),
+    }
+    Ok(())
+}
+
+pub async fn transaction_history(
+    address: &Pubkey,
+    mut limit: usize,
+    mut before: Option<Signature>,
+    until: Option<Signature>,
+    verbose: bool,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let bigtable = solana_storage_bigtable::LedgerStorage::new(true).await?;
+
+    while limit > 0 {
+        let results = bigtable
+            .get_confirmed_signatures_for_address(
+                address,
+                before.as_ref(),
+                until.as_ref(),
+                limit.min(1000),
+            )
+            .await?;
+
+        if results.is_empty() {
+            break;
+        }
+        before = Some(results.last().unwrap().signature);
+        assert!(limit >= results.len());
+        limit = limit.saturating_sub(results.len());
+
+        for result in results {
+            if verbose {
+                println!(
+                    "{}, slot={}, memo=\"{}\", status={}",
+                    result.signature,
+                    result.slot,
+                    result.memo.unwrap_or_else(|| "".to_string()),
+                    match result.err {
+                        None => "Confirmed".to_string(),
+                        Some(err) => format!("Failed: {:?}", err),
+                    }
+                );
+            } else {
+                println!("{}", result.signature);
+            }
+        }
+    }
+    Ok(())
+}
+
+pub trait BigTableSubCommand {
+    fn bigtable_subcommand(self) -> Self;
+}
+
+impl BigTableSubCommand for App<'_, '_> {
+    fn bigtable_subcommand(self) -> Self {
+        self.subcommand(
+            SubCommand::with_name("bigtable")
+                .about("Ledger data on a BigTable instance")
+                .setting(AppSettings::ArgRequiredElseHelp)
+                .subcommand(
+                    SubCommand::with_name("upload")
+                        .about("Upload the ledger to BigTable")
+                        .arg(
+                            Arg::with_name("starting_slot")
+                                .long("starting-slot")
+                                .validator(is_slot)
+                                .value_name("SLOT")
+                                .takes_value(true)
+                                .index(1)
+                                .help(
+                                    "Start uploading at this slot [default: first available slot]",
+                                ),
+                        )
+                        .arg(
+                            Arg::with_name("ending_slot")
+                                .long("ending-slot")
+                                .validator(is_slot)
+                                .value_name("SLOT")
+                                .takes_value(true)
+                                .index(2)
+                                .help("Stop uploading at this slot [default: last available slot]"),
+                        )
+                        .arg(
+                            Arg::with_name("allow_missing_metadata")
+                                .long("allow-missing-metadata")
+                                .takes_value(false)
+                                .help("Don't panic if transaction metadata is missing"),
+                        ),
+                )
+                .subcommand(
+                    SubCommand::with_name("first-available-block")
+                        .about("Get the first available block in the storage"),
+                )
+                .subcommand(
+                    SubCommand::with_name("blocks")
+                        .about("Get a list of slots with confirmed blocks for the given range")
+                        .arg(
+                            Arg::with_name("starting_slot")
+                                .long("starting-slot")
+                                .validator(is_slot)
+                                .value_name("SLOT")
+                                .takes_value(true)
+                                .index(1)
+                                .required(true)
+                                .default_value("0")
+                                .help("Start listing at this slot"),
+                        )
+                        .arg(
+                            Arg::with_name("limit")
+                                .long("limit")
+                                .validator(is_slot)
+                                .value_name("LIMIT")
+                                .takes_value(true)
+                                .index(2)
+                                .required(true)
+                                .default_value("1000")
+                                .help("Maximum number of slots to return"),
+                        ),
+                )
+                .subcommand(
+                    SubCommand::with_name("block")
+                        .about("Get a confirmed block")
+                        .arg(
+                            Arg::with_name("slot")
+                                .long("slot")
+                                .validator(is_slot)
+                                .value_name("SLOT")
+                                .takes_value(true)
+                                .index(1)
+                                .required(true),
+                        ),
+                )
+                .subcommand(
+                    SubCommand::with_name("confirm")
+                        .about("Confirm transaction by signature")
+                        .arg(
+                            Arg::with_name("signature")
+                                .long("signature")
+                                .value_name("TRANSACTION_SIGNATURE")
+                                .takes_value(true)
+                                .required(true)
+                                .index(1)
+                                .help("The transaction signature to confirm"),
+                        )
+                        .arg(
+                            Arg::with_name("verbose")
+                                .short("v")
+                                .long("verbose")
+                                .takes_value(false)
+                                .help("Show additional information"),
+                        ),
+                )
+                .subcommand(
+                    SubCommand::with_name("transaction-history")
+                        .about(
+                            "Show historical transactions affecting the given address \
+                             from newest to oldest",
+                        )
+                        .arg(
+                            Arg::with_name("address")
+                                .index(1)
+                                .value_name("ADDRESS")
+                                .required(true)
+                                .validator(is_valid_pubkey)
+                                .help("Account address"),
+                        )
+                        .arg(
+                            Arg::with_name("limit")
+                                .long("limit")
+                                .takes_value(true)
+                                .value_name("LIMIT")
+                                .validator(is_slot)
+                                .index(2)
+                                .default_value("18446744073709551615")
+                                .help("Maximum number of transaction signatures to return"),
+                        )
+                        .arg(
+                            Arg::with_name("before")
+                                .long("before")
+                                .value_name("TRANSACTION_SIGNATURE")
+                                .takes_value(true)
+                                .help("Start with the first signature older than this one"),
+                        )
+                        .arg(
+                            Arg::with_name("until")
+                                .long("until")
+                                .value_name("TRANSACTION_SIGNATURE")
+                                .takes_value(true)
+                                .help("End with the last signature newer than this one"),
+                        )
+                        .arg(
+                            Arg::with_name("verbose")
+                                .short("v")
+                                .long("verbose")
+                                .takes_value(false)
+                                .help("Show additional information"),
+                        ),
+                ),
+        )
+    }
+}
+
+pub fn bigtable_process_command(ledger_path: &Path, matches: &ArgMatches<'_>) {
+    let mut runtime = tokio::runtime::Runtime::new().unwrap();
+
+    let future = match matches.subcommand() {
+        ("upload", Some(arg_matches)) => {
+            let starting_slot = value_t!(arg_matches, "starting_slot", Slot).unwrap_or(0);
+            let ending_slot = value_t!(arg_matches, "ending_slot", Slot).ok();
+            let allow_missing_metadata = arg_matches.is_present("allow_missing_metadata");
+            let blockstore =
+                crate::open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary, None);
+
+            runtime.block_on(upload(
+                blockstore,
+                starting_slot,
+                ending_slot,
+                allow_missing_metadata,
+            ))
+        }
+        ("first-available-block", Some(_arg_matches)) => runtime.block_on(first_available_block()),
+        ("block", Some(arg_matches)) => {
+            let slot = value_t_or_exit!(arg_matches, "slot", Slot);
+            runtime.block_on(block(slot))
+        }
+        ("blocks", Some(arg_matches)) => {
+            let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
+            let limit = value_t_or_exit!(arg_matches, "limit", usize);
+
+            runtime.block_on(blocks(starting_slot, limit))
+        }
+        ("confirm", Some(arg_matches)) => {
+            let signature = arg_matches
+                .value_of("signature")
+                .unwrap()
+                .parse()
+                .expect("Invalid signature");
+            let verbose = arg_matches.is_present("verbose");
+
+            runtime.block_on(confirm(&signature, verbose))
+        }
+        ("transaction-history", Some(arg_matches)) => {
+            let address = pubkey_of(arg_matches, "address").unwrap();
+            let limit = value_t_or_exit!(arg_matches, "limit", usize);
+            let before = arg_matches
+                .value_of("before")
+                .map(|signature| signature.parse().expect("Invalid signature"));
+            let until = arg_matches
+                .value_of("until")
+                .map(|signature| signature.parse().expect("Invalid signature"));
+            let verbose = arg_matches.is_present("verbose");
+
+            runtime.block_on(transaction_history(&address, limit, before, until, verbose))
+        }
+        _ => unreachable!(),
+    };
+
+    future.unwrap_or_else(|err| {
+        eprintln!("{:?}", err);
+        exit(1);
+    });
+}

ledger-tool/src/main.rs

@@ -2,6 +2,7 @@ use clap::{
     crate_description, crate_name, value_t, value_t_or_exit, values_t_or_exit, App, Arg,
     ArgMatches, SubCommand,
 };
+use log::*;
 use regex::Regex;
 use serde_json::json;
 use solana_clap_utils::input_validators::{is_parsable, is_slot};
@@ -11,7 +12,7 @@ use solana_ledger::{
     bank_forks::{BankForks, SnapshotConfig},
     bank_forks_utils,
     blockstore::Blockstore,
-    blockstore_db::{self, AccessType, Column, Database},
+    blockstore_db::{self, AccessType, BlockstoreRecoveryMode, Column, Database},
     blockstore_processor::ProcessOptions,
     hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
     rooted_slot_iterator::RootedSlotIterator,
@@ -40,7 +41,8 @@ use std::{
     sync::Arc,
 };
 
-use log::*;
+mod bigtable;
+use bigtable::*;
 
 #[derive(PartialEq)]
 enum LedgerOutputMethod {
@@ -530,8 +532,12 @@ fn analyze_storage(database: &Database) -> Result<(), String> {
     Ok(())
 }
 
-fn open_blockstore(ledger_path: &Path, access_type: AccessType) -> Blockstore {
-    match Blockstore::open_with_access_type(ledger_path, access_type) {
+fn open_blockstore(
+    ledger_path: &Path,
+    access_type: AccessType,
+    wal_recovery_mode: Option<BlockstoreRecoveryMode>,
+) -> Blockstore {
+    match Blockstore::open_with_access_type(ledger_path, access_type, wal_recovery_mode) {
         Ok(blockstore) => blockstore,
         Err(err) => {
             eprintln!("Failed to open ledger at {:?}: {:?}", ledger_path, err);
@@ -541,7 +547,7 @@ fn open_blockstore(ledger_path: &Path, access_type: AccessType) -> Blockstore {
 }
 
 fn open_database(ledger_path: &Path, access_type: AccessType) -> Database {
-    match Database::open(&ledger_path.join("rocksdb"), access_type) {
+    match Database::open(&ledger_path.join("rocksdb"), access_type, None) {
         Ok(database) => database,
         Err(err) => {
             eprintln!("Unable to read the Ledger rocksdb: {:?}", err);
@@ -565,8 +571,9 @@ fn load_bank_forks(
     genesis_config: &GenesisConfig,
     process_options: ProcessOptions,
     access_type: AccessType,
+    wal_recovery_mode: Option<BlockstoreRecoveryMode>,
 ) -> bank_forks_utils::LoadResult {
-    let blockstore = open_blockstore(&ledger_path, access_type);
+    let blockstore = open_blockstore(&ledger_path, access_type, wal_recovery_mode);
     let snapshot_path = ledger_path.clone().join(if blockstore.is_primary_access() {
         "snapshot"
     } else {
@@ -704,6 +711,22 @@ fn main() {
                 .global(true)
                 .help("Use DIR for ledger location"),
         )
+        .bigtable_subcommand()
+        .arg(
+            Arg::with_name("wal_recovery_mode")
+                .long("wal-recovery-mode")
+                .value_name("MODE")
+                .takes_value(true)
+                .global(true)
+                .possible_values(&[
+                    "tolerate_corrupted_tail_records",
+                    "absolute_consistency",
+                    "point_in_time",
+                    "skip_any_corrupted_record"])
+                .help(
+                    "Mode to recovery the ledger db write ahead log."
+                ),
+        )
         .subcommand(
             SubCommand::with_name("print")
             .about("Print the ledger")
@@ -974,12 +997,21 @@ fn main() {
         exit(1);
     });
 
+    let wal_recovery_mode = matches
+        .value_of("wal_recovery_mode")
+        .map(BlockstoreRecoveryMode::from);
+
     match matches.subcommand() {
+        ("bigtable", Some(arg_matches)) => bigtable_process_command(&ledger_path, arg_matches),
         ("print", Some(arg_matches)) => {
             let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
             let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
             output_ledger(
-                open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary),
+                open_blockstore(
+                    &ledger_path,
+                    AccessType::TryPrimaryThenSecondary,
+                    wal_recovery_mode,
+                ),
                 starting_slot,
                 allow_dead_slots,
                 LedgerOutputMethod::Print,
@@ -1008,6 +1040,7 @@ fn main() {
                 &genesis_config,
                 process_options,
                 AccessType::TryPrimaryThenSecondary,
+                wal_recovery_mode,
             ) {
                 Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
                     println!(
@@ -1027,7 +1060,11 @@ fn main() {
         ("slot", Some(arg_matches)) => {
             let slots = values_t_or_exit!(arg_matches, "slots", Slot);
             let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
-            let blockstore = open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary);
+            let blockstore = open_blockstore(
+                &ledger_path,
+                AccessType::TryPrimaryThenSecondary,
+                wal_recovery_mode,
+            );
             for slot in slots {
                 println!("Slot {}", slot);
                 if let Err(err) = output_slot(
@@ -1044,7 +1081,11 @@ fn main() {
             let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
             let allow_dead_slots = arg_matches.is_present("allow_dead_slots");
             output_ledger(
-                open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary),
+                open_blockstore(
+                    &ledger_path,
+                    AccessType::TryPrimaryThenSecondary,
+                    wal_recovery_mode,
+                ),
                 starting_slot,
                 allow_dead_slots,
                 LedgerOutputMethod::Json,
@@ -1052,7 +1093,8 @@ fn main() {
         }
         ("set-dead-slot", Some(arg_matches)) => {
             let slots = values_t_or_exit!(arg_matches, "slots", Slot);
-            let blockstore = open_blockstore(&ledger_path, AccessType::PrimaryOnly);
+            let blockstore =
+                open_blockstore(&ledger_path, AccessType::PrimaryOnly, wal_recovery_mode);
             for slot in slots {
                 match blockstore.set_dead_slot(slot) {
                     Ok(_) => println!("Slot {} dead", slot),
@@ -1063,7 +1105,11 @@ fn main() {
         ("parse_full_frozen", Some(arg_matches)) => {
             let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot);
             let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot);
-            let blockstore = open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary);
+            let blockstore = open_blockstore(
+                &ledger_path,
+                AccessType::TryPrimaryThenSecondary,
+                wal_recovery_mode,
+            );
             let mut ancestors = BTreeSet::new();
             if blockstore.meta(ending_slot).unwrap().is_none() {
                 panic!("Ending slot doesn't exist");
@@ -1142,6 +1188,7 @@ fn main() {
                 &open_genesis_config_by(&ledger_path, arg_matches),
                 process_options,
                 AccessType::TryPrimaryThenSecondary,
+                wal_recovery_mode,
             )
             .unwrap_or_else(|err| {
                 eprintln!("Ledger verification failed: {:?}", err);
@@ -1165,6 +1212,7 @@ fn main() {
                 &open_genesis_config_by(&ledger_path, arg_matches),
                 process_options,
                 AccessType::TryPrimaryThenSecondary,
+                wal_recovery_mode,
             ) {
                 Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
                     let dot = graph_forks(&bank_forks, arg_matches.is_present("include_all_votes"));
@@ -1216,6 +1264,7 @@ fn main() {
                 &genesis_config,
                 process_options,
                 AccessType::TryPrimaryThenSecondary,
+                wal_recovery_mode,
             ) {
                 Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
                     let bank = bank_forks
@@ -1310,6 +1359,7 @@ fn main() {
                 &genesis_config,
                 process_options,
                 AccessType::TryPrimaryThenSecondary,
+                wal_recovery_mode,
             ) {
                 Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
                     let slot = bank_forks.working_bank().slot();
@@ -1358,6 +1408,7 @@ fn main() {
                 &genesis_config,
                 process_options,
                 AccessType::TryPrimaryThenSecondary,
+                wal_recovery_mode,
             ) {
                 Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
                     let slot = bank_forks.working_bank().slot();
@@ -1468,12 +1519,17 @@ fn main() {
         ("purge", Some(arg_matches)) => {
             let start_slot = value_t_or_exit!(arg_matches, "start_slot", Slot);
             let end_slot = value_t_or_exit!(arg_matches, "end_slot", Slot);
-            let blockstore = open_blockstore(&ledger_path, AccessType::PrimaryOnly);
+            let blockstore =
+                open_blockstore(&ledger_path, AccessType::PrimaryOnly, wal_recovery_mode);
             blockstore.purge_and_compact_slots(start_slot, end_slot);
             blockstore.purge_from_next_slots(start_slot, end_slot);
         }
         ("list-roots", Some(arg_matches)) => {
-            let blockstore = open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary);
+            let blockstore = open_blockstore(
+                &ledger_path,
+                AccessType::TryPrimaryThenSecondary,
+                wal_recovery_mode,
+            );
             let max_height = if let Some(height) = arg_matches.value_of("max_height") {
                 usize::from_str(height).expect("Maximum height must be a number")
             } else {
@@ -1526,8 +1582,12 @@ fn main() {
                 });
         }
         ("bounds", Some(arg_matches)) => {
-            match open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary)
-                .slot_meta_iterator(0)
+            match open_blockstore(
+                &ledger_path,
+                AccessType::TryPrimaryThenSecondary,
+                wal_recovery_mode,
+            )
+            .slot_meta_iterator(0)
             {
                 Ok(metas) => {
                     let all = arg_matches.is_present("all");

ledger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-ledger"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana ledger"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -32,19 +32,19 @@ reed-solomon-erasure = { version = "4.0.2", features = ["simd-accel"] }
 regex = "1.3.7"
 serde = "1.0.110"
 serde_bytes = "0.11.4"
-solana-transaction-status = { path = "../transaction-status", version = "1.2.19" }
-solana-genesis-programs = { path = "../genesis-programs", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-measure = { path = "../measure", version = "1.2.19" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.2.19" }
-solana-metrics = { path = "../metrics", version = "1.2.19" }
-solana-perf = { path = "../perf", version = "1.2.19" }
+solana-transaction-status = { path = "../transaction-status", version = "1.2.24" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-measure = { path = "../measure", version = "1.2.24" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.2.24" }
+solana-metrics = { path = "../metrics", version = "1.2.24" }
+solana-perf = { path = "../perf", version = "1.2.24" }
 ed25519-dalek = "1.0.0-pre.3"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.19" }
-solana-runtime = { path = "../runtime", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-stake-program = { path = "../programs/stake", version = "1.2.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.19" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.24" }
+solana-runtime = { path = "../runtime", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.24" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.24" }
 symlink = "0.1.0"
 tar = "0.4.28"
 thiserror = "1.0"
@@ -62,7 +62,7 @@ features = ["lz4"]
 [dev-dependencies]
 assert_matches = "1.3.0"
 matches = "0.1.6"
-solana-budget-program = { path = "../programs/budget", version = "1.2.19" }
+solana-budget-program = { path = "../programs/budget", version = "1.2.24" }
 
 [lib]
 crate-type = ["lib"]

ledger/src/blockstore.rs

@@ -4,8 +4,8 @@
 pub use crate::{blockstore_db::BlockstoreError, blockstore_meta::SlotMeta};
 use crate::{
     blockstore_db::{
-        columns as cf, AccessType, Column, Database, IteratorDirection, IteratorMode, LedgerColumn,
-        Result, WriteBatch,
+        columns as cf, AccessType, BlockstoreRecoveryMode, Column, Database, IteratorDirection,
+        IteratorMode, LedgerColumn, Result, WriteBatch,
     },
     blockstore_meta::*,
     entry::{create_ticks, Entry},
@@ -232,17 +232,22 @@ impl Blockstore {
 
     /// Opens a Ledger in directory, provides "infinite" window of shreds
     pub fn open(ledger_path: &Path) -> Result<Blockstore> {
-        Self::do_open(ledger_path, AccessType::PrimaryOnly)
+        Self::do_open(ledger_path, AccessType::PrimaryOnly, None)
     }
 
     pub fn open_with_access_type(
         ledger_path: &Path,
         access_type: AccessType,
+        recovery_mode: Option<BlockstoreRecoveryMode>,
     ) -> Result<Blockstore> {
-        Self::do_open(ledger_path, access_type)
+        Self::do_open(ledger_path, access_type, recovery_mode)
     }
 
-    fn do_open(ledger_path: &Path, access_type: AccessType) -> Result<Blockstore> {
+    fn do_open(
+        ledger_path: &Path,
+        access_type: AccessType,
+        recovery_mode: Option<BlockstoreRecoveryMode>,
+    ) -> Result<Blockstore> {
         fs::create_dir_all(&ledger_path)?;
         let blockstore_path = ledger_path.join(BLOCKSTORE_DIRECTORY);
 
@@ -251,7 +256,7 @@ impl Blockstore {
         // Open the database
         let mut measure = Measure::start("open");
         info!("Opening database at {:?}", blockstore_path);
-        let db = Database::open(&blockstore_path, access_type)?;
+        let db = Database::open(&blockstore_path, access_type, recovery_mode)?;
 
         // Create the metadata column family
         let meta_cf = db.column();
@@ -332,8 +337,10 @@ impl Blockstore {
 
     pub fn open_with_signal(
         ledger_path: &Path,
+        recovery_mode: Option<BlockstoreRecoveryMode>,
     ) -> Result<(Self, Receiver<bool>, CompletedSlotsReceiver)> {
-        let mut blockstore = Self::open_with_access_type(ledger_path, AccessType::PrimaryOnly)?;
+        let mut blockstore =
+            Self::open_with_access_type(ledger_path, AccessType::PrimaryOnly, recovery_mode)?;
         let (signal_sender, signal_receiver) = sync_channel(1);
         let (completed_slots_sender, completed_slots_receiver) =
             sync_channel(MAX_COMPLETED_SLOTS_IN_CHANNEL);
@@ -1652,7 +1659,7 @@ impl Blockstore {
         iterator
             .map(|transaction| {
                 let signature = transaction.signatures[0];
-                let encoded_transaction = EncodedTransaction::encode(transaction, encoding.clone());
+                let encoded_transaction = EncodedTransaction::encode(transaction, encoding);
                 TransactionWithStatusMeta {
                     transaction: encoded_transaction,
                     meta: self
@@ -1664,6 +1671,10 @@ impl Blockstore {
             .collect()
     }
 
+    /// Initializes the TransactionStatusIndex column family with two records, `0` and `1`,
+    /// which are used as the primary index for entries in the TransactionStatus and
+    /// AddressSignatures columns. At any given time, one primary index is active (ie. new records
+    /// are stored under this index), the other is frozen.
     fn initialize_transaction_status_index(&self) -> Result<()> {
         self.transaction_status_index_cf
             .put(0, &TransactionStatusIndexMeta::default())?;
@@ -1680,6 +1691,8 @@ impl Blockstore {
         )
     }
 
+    /// Toggles the active primary index between `0` and `1`, and clears the stored max-slot of the
+    /// frozen index in preparation for pruning.
     fn toggle_transaction_status_index(
         &self,
         batch: &mut WriteBatch,
@@ -1797,9 +1810,9 @@ impl Blockstore {
                 (transaction_status_cf_primary_index, signature, 0),
                 IteratorDirection::Forward,
             ))?;
-            for ((_, sig, slot), data) in index_iterator {
+            for ((i, sig, slot), data) in index_iterator {
                 counter += 1;
-                if sig != signature {
+                if i != transaction_status_cf_primary_index || sig != signature {
                     break;
                 }
                 if self.is_root(slot) {
@@ -1835,8 +1848,9 @@ impl Blockstore {
             ("method", "get_confirmed_transaction".to_string(), String)
         );
         if let Some((slot, status)) = self.get_transaction_status(signature.clone())? {
-            let transaction = self.find_transaction_in_slot(slot, signature)?
-                .expect("Transaction to exist in slot entries if it exists in statuses and hasn't been cleaned up");
+            let transaction = self
+                .find_transaction_in_slot(slot, signature)?
+                .ok_or(BlockstoreError::TransactionStatusSlotMismatch)?; // Should not happen
             let encoding = encoding.unwrap_or(UiTransactionEncoding::Json);
             let encoded_transaction = EncodedTransaction::encode(transaction, encoding);
             Ok(Some(ConfirmedTransaction {
@@ -1894,7 +1908,7 @@ impl Blockstore {
                 }
             }
         }
-        signatures.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
+        signatures.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap().then(a.1.cmp(&b.1)));
         Ok(signatures)
     }
 
@@ -1921,6 +1935,7 @@ impl Blockstore {
         address: Pubkey,
         highest_confirmed_root: Slot,
         before: Option<Signature>,
+        until: Option<Signature>,
         limit: usize,
     ) -> Result<Vec<ConfirmedTransactionStatusWithSignature>> {
         datapoint_info!(
@@ -1935,7 +1950,8 @@ impl Blockstore {
         // Figure the `slot` to start listing signatures at, based on the ledger location of the
         // `before` signature if present.  Also generate a HashSet of signatures that should
         // be excluded from the results.
-        let (mut slot, mut excluded_signatures) = match before {
+        let mut get_before_slot_timer = Measure::start("get_before_slot_timer");
+        let (slot, mut before_excluded_signatures) = match before {
             None => (highest_confirmed_root, None),
             Some(before) => {
                 let transaction_status = self.get_transaction_status(before)?;
@@ -1967,9 +1983,10 @@ impl Blockstore {
                             .collect();
 
                         // Sort signatures as a way to entire a stable ordering within a slot, as
-                        // `self.find_address_signatures()` is ordered by signatures ordered and
+                        // the AddressSignatures column is ordered by signatures within a slot,
                         // not by block ordering
                         slot_signatures.sort();
+                        slot_signatures.reverse();
 
                         if let Some(pos) = slot_signatures.iter().position(|&x| x == before) {
                             slot_signatures.truncate(pos + 1);
@@ -1983,36 +2000,160 @@ impl Blockstore {
                 }
             }
         };
+        get_before_slot_timer.stop();
+
+        // Generate a HashSet of signatures that should be excluded from the results based on
+        // `until` signature
+        let mut get_until_slot_timer = Measure::start("get_until_slot_timer");
+        let (lowest_slot, until_excluded_signatures) = match until {
+            None => (0, HashSet::new()),
+            Some(until) => {
+                let transaction_status = self.get_transaction_status(until)?;
+                match transaction_status {
+                    None => (0, HashSet::new()),
+                    Some((slot, _)) => {
+                        let confirmed_block = self
+                            .get_confirmed_block(slot, Some(UiTransactionEncoding::Binary))
+                            .map_err(|err| {
+                                BlockstoreError::IO(IOError::new(
+                                    ErrorKind::Other,
+                                    format!("Unable to get confirmed block: {}", err),
+                                ))
+                            })?;
+
+                        // Load all signatures for the block
+                        let mut slot_signatures: Vec<_> = confirmed_block
+                            .transactions
+                            .iter()
+                            .filter_map(|transaction_with_meta| {
+                                if let Some(transaction) =
+                                    transaction_with_meta.transaction.decode()
+                                {
+                                    transaction.signatures.into_iter().next()
+                                } else {
+                                    None
+                                }
+                            })
+                            .collect();
+
+                        // Sort signatures as a way to entire a stable ordering within a slot, as
+                        // the AddressSignatures column is ordered by signatures within a slot,
+                        // not by block ordering
+                        slot_signatures.sort();
+                        slot_signatures.reverse();
+
+                        if let Some(pos) = slot_signatures.iter().position(|&x| x == until) {
+                            slot_signatures = slot_signatures.split_off(pos);
+                        }
+
+                        (slot, slot_signatures.into_iter().collect::<HashSet<_>>())
+                    }
+                }
+            }
+        };
+        get_until_slot_timer.stop();
 
         // Fetch the list of signatures that affect the given address
         let first_available_block = self.get_first_available_block()?;
         let mut address_signatures = vec![];
-        loop {
-            if address_signatures.len() >= limit {
-                address_signatures.truncate(limit);
+
+        // Get signatures in `slot`
+        let mut get_initial_slot_timer = Measure::start("get_initial_slot_timer");
+        let mut signatures = self.find_address_signatures(address, slot, slot)?;
+        signatures.reverse();
+        if let Some(excluded_signatures) = before_excluded_signatures.take() {
+            address_signatures.extend(
+                signatures
+                    .into_iter()
+                    .filter(|(_, signature)| !excluded_signatures.contains(&signature)),
+            )
+        } else {
+            address_signatures.append(&mut signatures);
+        }
+        get_initial_slot_timer.stop();
+
+        // Check the active_transaction_status_index to see if it contains slot. If so, start with
+        // that index, as it will contain higher slots
+        let starting_primary_index = *self.active_transaction_status_index.read().unwrap();
+        let next_primary_index = if starting_primary_index == 0 { 1 } else { 0 };
+        let next_max_slot = self
+            .transaction_status_index_cf
+            .get(next_primary_index)?
+            .unwrap()
+            .max_slot;
+
+        let mut starting_primary_index_iter_timer = Measure::start("starting_primary_index_iter");
+        if slot > next_max_slot {
+            let mut starting_iterator = self.address_signatures_cf.iter(IteratorMode::From(
+                (starting_primary_index, address, slot, Signature::default()),
+                IteratorDirection::Reverse,
+            ))?;
+
+            // Iterate through starting_iterator until limit is reached
+            while address_signatures.len() < limit {
+                if let Some(((i, key_address, slot, signature), _)) = starting_iterator.next() {
+                    if slot == next_max_slot || slot < lowest_slot {
+                        break;
+                    }
+                    if i == starting_primary_index
+                        && key_address == address
+                        && slot >= first_available_block
+                    {
+                        if self.is_root(slot) {
+                            address_signatures.push((slot, signature));
+                        }
+                        continue;
+                    }
+                }
                 break;
             }
 
-            let mut signatures = self.find_address_signatures(address, slot, slot)?;
-            if let Some(excluded_signatures) = excluded_signatures.take() {
-                address_signatures.extend(
-                    signatures
-                        .into_iter()
-                        .filter(|(_, signature)| !excluded_signatures.contains(&signature)),
-                )
-            } else {
+            // Handle slots that cross primary indexes
+            if next_max_slot >= lowest_slot {
+                let mut signatures =
+                    self.find_address_signatures(address, next_max_slot, next_max_slot)?;
+                signatures.reverse();
                 address_signatures.append(&mut signatures);
             }
-            excluded_signatures = None;
-
-            if slot == first_available_block {
-                break;
-            }
-            slot -= 1;
         }
+        starting_primary_index_iter_timer.stop();
+
+        // Iterate through next_iterator until limit is reached
+        let mut next_primary_index_iter_timer = Measure::start("next_primary_index_iter_timer");
+        let mut next_iterator = self.address_signatures_cf.iter(IteratorMode::From(
+            (next_primary_index, address, slot, Signature::default()),
+            IteratorDirection::Reverse,
+        ))?;
+        while address_signatures.len() < limit {
+            if let Some(((i, key_address, slot, signature), _)) = next_iterator.next() {
+                // Skip next_max_slot, which is already included
+                if slot == next_max_slot {
+                    continue;
+                }
+                if slot < lowest_slot {
+                    break;
+                }
+                if i == next_primary_index
+                    && key_address == address
+                    && slot >= first_available_block
+                {
+                    if self.is_root(slot) {
+                        address_signatures.push((slot, signature));
+                    }
+                    continue;
+                }
+            }
+            break;
+        }
+        next_primary_index_iter_timer.stop();
+        let mut address_signatures: Vec<(Slot, Signature)> = address_signatures
+            .into_iter()
+            .filter(|(_, signature)| !until_excluded_signatures.contains(&signature))
+            .collect();
         address_signatures.truncate(limit);
 
         // Fill in the status information for each found transaction
+        let mut get_status_info_timer = Measure::start("get_status_info_timer");
         let mut infos = vec![];
         for (slot, signature) in address_signatures.into_iter() {
             let transaction_status = self.get_transaction_status(signature)?;
@@ -2027,6 +2168,41 @@ impl Blockstore {
                 memo: None,
             });
         }
+        get_status_info_timer.stop();
+
+        datapoint_info!(
+            "blockstore-get-conf-sigs-for-addr-2",
+            (
+                "get_before_slot_us",
+                get_before_slot_timer.as_us() as i64,
+                i64
+            ),
+            (
+                "get_initial_slot_us",
+                get_initial_slot_timer.as_us() as i64,
+                i64
+            ),
+            (
+                "starting_primary_index_iter_us",
+                starting_primary_index_iter_timer.as_us() as i64,
+                i64
+            ),
+            (
+                "next_primary_index_iter_us",
+                next_primary_index_iter_timer.as_us() as i64,
+                i64
+            ),
+            (
+                "get_status_info_us",
+                get_status_info_timer.as_us() as i64,
+                i64
+            ),
+            (
+                "get_until_slot_us",
+                get_until_slot_timer.as_us() as i64,
+                i64
+            )
+        );
 
         Ok(infos)
     }
@@ -2836,7 +3012,7 @@ pub fn create_new_ledger(
     genesis_config.write(&ledger_path)?;
 
     // Fill slot 0 with ticks that link back to the genesis_config to bootstrap the ledger.
-    let blockstore = Blockstore::open_with_access_type(ledger_path, access_type)?;
+    let blockstore = Blockstore::open_with_access_type(ledger_path, access_type, None)?;
     let ticks_per_slot = genesis_config.ticks_per_slot;
     let hashes_per_tick = genesis_config.poh_config.hashes_per_tick.unwrap_or(0);
     let entries = create_ticks(ticks_per_slot, hashes_per_tick, genesis_config.hash());
@@ -3773,7 +3949,7 @@ pub mod tests {
     pub fn test_new_shreds_signal() {
         // Initialize ledger
         let ledger_path = get_tmp_ledger_path!();
-        let (ledger, recvr, _) = Blockstore::open_with_signal(&ledger_path).unwrap();
+        let (ledger, recvr, _) = Blockstore::open_with_signal(&ledger_path, None).unwrap();
         let ledger = Arc::new(ledger);
 
         let entries_per_slot = 50;
@@ -3853,7 +4029,7 @@ pub mod tests {
     pub fn test_completed_shreds_signal() {
         // Initialize ledger
         let ledger_path = get_tmp_ledger_path!();
-        let (ledger, _, recvr) = Blockstore::open_with_signal(&ledger_path).unwrap();
+        let (ledger, _, recvr) = Blockstore::open_with_signal(&ledger_path, None).unwrap();
         let ledger = Arc::new(ledger);
 
         let entries_per_slot = 10;
@@ -3875,7 +4051,7 @@ pub mod tests {
     pub fn test_completed_shreds_signal_orphans() {
         // Initialize ledger
         let ledger_path = get_tmp_ledger_path!();
-        let (ledger, _, recvr) = Blockstore::open_with_signal(&ledger_path).unwrap();
+        let (ledger, _, recvr) = Blockstore::open_with_signal(&ledger_path, None).unwrap();
         let ledger = Arc::new(ledger);
 
         let entries_per_slot = 10;
@@ -3915,7 +4091,7 @@ pub mod tests {
     pub fn test_completed_shreds_signal_many() {
         // Initialize ledger
         let ledger_path = get_tmp_ledger_path!();
-        let (ledger, _, recvr) = Blockstore::open_with_signal(&ledger_path).unwrap();
+        let (ledger, _, recvr) = Blockstore::open_with_signal(&ledger_path, None).unwrap();
         let ledger = Arc::new(ledger);
 
         let entries_per_slot = 10;
@@ -6059,6 +6235,19 @@ pub mod tests {
         }
     }
 
+    #[test]
+    fn test_empty_transaction_status() {
+        let blockstore_path = get_tmp_ledger_path!();
+        let blockstore = Blockstore::open(&blockstore_path).unwrap();
+        blockstore.set_roots(&[0]).unwrap();
+        assert_eq!(
+            blockstore
+                .get_confirmed_transaction(Signature::default(), None)
+                .unwrap(),
+            None
+        );
+    }
+
     #[test]
     fn test_get_confirmed_signatures_for_address() {
         let blockstore_path = get_tmp_ledger_path!();
@@ -6225,14 +6414,18 @@ pub mod tests {
             let address0 = Pubkey::new_rand();
             let address1 = Pubkey::new_rand();
 
-            for slot in 2..=4 {
+            for slot in 2..=8 {
                 let entries = make_slot_entries_with_transaction_addresses(&[
                     address0, address1, address0, address1,
                 ]);
                 let shreds = entries_to_test_shreds(entries.clone(), slot, slot - 1, true, 0);
                 blockstore.insert_shreds(shreds, None, false).unwrap();
 
-                for entry in &entries {
+                for (i, entry) in entries.iter().enumerate() {
+                    if slot == 4 && i == 2 {
+                        // Purge to freeze index 0 and write address-signatures in new primary index
+                        blockstore.run_purge(0, 1, PurgeType::PrimaryIndex).unwrap();
+                    }
                     for transaction in &entry.transactions {
                         assert_eq!(transaction.signatures.len(), 1);
                         blockstore
@@ -6247,8 +6440,9 @@ pub mod tests {
                     }
                 }
             }
-            blockstore.set_roots(&[1, 2, 3, 4]).unwrap();
-            let highest_confirmed_root = 4;
+            // Leave one slot unrooted to test only returns confirmed signatures
+            blockstore.set_roots(&[1, 2, 4, 5, 6, 7, 8]).unwrap();
+            let highest_confirmed_root = 8;
 
             // Fetch all signatures for address 0 at once...
             let all0 = blockstore
@@ -6256,10 +6450,11 @@ pub mod tests {
                     address0,
                     highest_confirmed_root,
                     None,
+                    None,
                     usize::MAX,
                 )
                 .unwrap();
-            assert_eq!(all0.len(), 6);
+            assert_eq!(all0.len(), 12);
 
             // Fetch all signatures for address 1 at once...
             let all1 = blockstore
@@ -6267,10 +6462,11 @@ pub mod tests {
                     address1,
                     highest_confirmed_root,
                     None,
+                    None,
                     usize::MAX,
                 )
                 .unwrap();
-            assert_eq!(all1.len(), 6);
+            assert_eq!(all1.len(), 12);
 
             assert!(all0 != all1);
 
@@ -6285,23 +6481,58 @@ pub mod tests {
                         } else {
                             Some(all0[i - 1].signature)
                         },
+                        None,
                         1,
                     )
                     .unwrap();
                 assert_eq!(results.len(), 1);
                 assert_eq!(results[0], all0[i], "Unexpected result for {}", i);
             }
+            // Fetch all signatures for address 0 individually using `until`
+            for i in 0..all0.len() {
+                let results = blockstore
+                    .get_confirmed_signatures_for_address2(
+                        address0,
+                        highest_confirmed_root,
+                        if i == 0 {
+                            None
+                        } else {
+                            Some(all0[i - 1].signature)
+                        },
+                        if i == all0.len() - 1 || i == all0.len() {
+                            None
+                        } else {
+                            Some(all0[i + 1].signature)
+                        },
+                        10,
+                    )
+                    .unwrap();
+                assert_eq!(results.len(), 1);
+                assert_eq!(results[0], all0[i], "Unexpected result for {}", i);
+            }
 
             assert!(blockstore
                 .get_confirmed_signatures_for_address2(
                     address0,
                     highest_confirmed_root,
                     Some(all0[all0.len() - 1].signature),
+                    None,
                     1,
                 )
                 .unwrap()
                 .is_empty());
 
+            assert!(blockstore
+                .get_confirmed_signatures_for_address2(
+                    address0,
+                    highest_confirmed_root,
+                    None,
+                    Some(all0[0].signature),
+                    2,
+                )
+                .unwrap()
+                .is_empty());
+
             // Fetch all signatures for address 0, three at a time
             assert!(all0.len() % 3 == 0);
             for i in (0..all0.len()).step_by(3) {
@@ -6314,6 +6545,7 @@ pub mod tests {
                         } else {
                             Some(all0[i - 1].signature)
                         },
+                        None,
                         3,
                     )
                     .unwrap();
@@ -6323,7 +6555,7 @@ pub mod tests {
                 assert_eq!(results[2], all0[i + 2]);
             }
 
-            // Ensure that the signatures within a slot are ordered by signature
+            // Ensure that the signatures within a slot are reverse ordered by signature
             // (current limitation of the .get_confirmed_signatures_for_address2())
             for i in (0..all1.len()).step_by(2) {
                 let results = blockstore
@@ -6335,28 +6567,41 @@ pub mod tests {
                         } else {
                             Some(all1[i - 1].signature)
                         },
+                        None,
                         2,
                     )
                     .unwrap();
                 assert_eq!(results.len(), 2);
                 assert_eq!(results[0].slot, results[1].slot);
-                assert!(results[0].signature <= results[1].signature);
+                assert!(results[0].signature >= results[1].signature);
                 assert_eq!(results[0], all1[i]);
                 assert_eq!(results[1], all1[i + 1]);
             }
 
-            // A search for address 0 with a `before` signature from address1 should also work
+            // A search for address 0 with `before` and/or `until` signatures from address1 should also work
             let results = blockstore
                 .get_confirmed_signatures_for_address2(
                     address0,
                     highest_confirmed_root,
                     Some(all1[0].signature),
+                    None,
                     usize::MAX,
                 )
                 .unwrap();
             // The exact number of results returned is variable, based on the sort order of the
             // random signatures that are generated
             assert!(!results.is_empty());
+
+            let results2 = blockstore
+                .get_confirmed_signatures_for_address2(
+                    address0,
+                    highest_confirmed_root,
+                    Some(all1[0].signature),
+                    Some(all1[4].signature),
+                    usize::MAX,
+                )
+                .unwrap();
+            assert!(results2.len() < results.len());
         }
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }

ledger/src/blockstore/blockstore_purge.rs

@@ -235,6 +235,9 @@ impl Blockstore {
         Ok(result)
     }
 
+    /// Purges special columns (using a non-Slot primary-index) exactly, by deserializing each slot
+    /// being purged and iterating through all transactions to determine the keys of individual
+    /// records. **This method is very slow.**
     fn purge_special_columns_exact(
         &self,
         batch: &mut WriteBatch,
@@ -279,6 +282,8 @@ impl Blockstore {
         Ok(())
     }
 
+    /// Purges special columns (using a non-Slot primary-index) by range. Purge occurs if frozen
+    /// primary index has a max-slot less than the highest slot being purged.
     fn purge_special_columns_with_primary_index(
         &self,
         write_batch: &mut WriteBatch,

ledger/src/blockstore_db.rs

@@ -4,7 +4,7 @@ use byteorder::{BigEndian, ByteOrder};
 use log::*;
 pub use rocksdb::Direction as IteratorDirection;
 use rocksdb::{
-    self, ColumnFamily, ColumnFamilyDescriptor, DBIterator, DBRawIterator,
+    self, ColumnFamily, ColumnFamilyDescriptor, DBIterator, DBRawIterator, DBRecoveryMode,
     IteratorMode as RocksIteratorMode, Options, WriteBatch as RWriteBatch, DB,
 };
 use serde::de::DeserializeOwned;
@@ -39,7 +39,9 @@ const CODE_SHRED_CF: &str = "code_shred";
 const TRANSACTION_STATUS_CF: &str = "transaction_status";
 /// Column family for Address Signatures
 const ADDRESS_SIGNATURES_CF: &str = "address_signatures";
-/// Column family for Transaction Status Index
+/// Column family for the Transaction Status Index.
+/// This column family is used for tracking the active primary index for columns that for
+/// query performance reasons should not be indexed by Slot.
 const TRANSACTION_STATUS_INDEX_CF: &str = "transaction_status_index";
 /// Column family for Rewards
 const REWARDS_CF: &str = "rewards";
@@ -57,6 +59,7 @@ pub enum BlockstoreError {
     SlotCleanedUp,
     UnpackError(#[from] UnpackError),
     UnableToSetOpenFileDescriptorLimit,
+    TransactionStatusSlotMismatch,
 }
 pub type Result<T> = std::result::Result<T, BlockstoreError>;
 
@@ -137,11 +140,51 @@ pub enum ActualAccessType {
     Secondary,
 }
 
+#[derive(Debug, Clone)]
+pub enum BlockstoreRecoveryMode {
+    TolerateCorruptedTailRecords,
+    AbsoluteConsistency,
+    PointInTime,
+    SkipAnyCorruptedRecord,
+}
+
+impl From<&str> for BlockstoreRecoveryMode {
+    fn from(string: &str) -> Self {
+        match string {
+            "tolerate_corrupted_tail_records" => {
+                BlockstoreRecoveryMode::TolerateCorruptedTailRecords
+            }
+            "absolute_consistency" => BlockstoreRecoveryMode::AbsoluteConsistency,
+            "point_in_time" => BlockstoreRecoveryMode::PointInTime,
+            "skip_any_corrupted_record" => BlockstoreRecoveryMode::SkipAnyCorruptedRecord,
+            bad_mode => panic!("Invalid recovery mode: {}", bad_mode),
+        }
+    }
+}
+impl Into<DBRecoveryMode> for BlockstoreRecoveryMode {
+    fn into(self) -> DBRecoveryMode {
+        match self {
+            BlockstoreRecoveryMode::TolerateCorruptedTailRecords => {
+                DBRecoveryMode::TolerateCorruptedTailRecords
+            }
+            BlockstoreRecoveryMode::AbsoluteConsistency => DBRecoveryMode::AbsoluteConsistency,
+            BlockstoreRecoveryMode::PointInTime => DBRecoveryMode::PointInTime,
+            BlockstoreRecoveryMode::SkipAnyCorruptedRecord => {
+                DBRecoveryMode::SkipAnyCorruptedRecord
+            }
+        }
+    }
+}
+
 #[derive(Debug)]
 struct Rocks(rocksdb::DB, ActualAccessType);
 
 impl Rocks {
-    fn open(path: &Path, access_type: AccessType) -> Result<Rocks> {
+    fn open(
+        path: &Path,
+        access_type: AccessType,
+        recovery_mode: Option<BlockstoreRecoveryMode>,
+    ) -> Result<Rocks> {
         use columns::{
             AddressSignatures, DeadSlots, DuplicateSlots, ErasureMeta, Index, Orphans, Rewards,
             Root, ShredCode, ShredData, SlotMeta, TransactionStatus, TransactionStatusIndex,
@@ -151,6 +194,9 @@ impl Rocks {
 
         // Use default database options
         let mut db_options = get_db_options();
+        if let Some(recovery_mode) = recovery_mode {
+            db_options.set_wal_recovery_mode(recovery_mode.into());
+        }
 
         // Column family names
         let meta_cf_descriptor = ColumnFamilyDescriptor::new(SlotMeta::NAME, get_cf_options());
@@ -626,8 +672,12 @@ pub struct WriteBatch<'a> {
 }
 
 impl Database {
-    pub fn open(path: &Path, access_type: AccessType) -> Result<Self> {
-        let backend = Arc::new(Rocks::open(path, access_type)?);
+    pub fn open(
+        path: &Path,
+        access_type: AccessType,
+        recovery_mode: Option<BlockstoreRecoveryMode>,
+    ) -> Result<Self> {
+        let backend = Arc::new(Rocks::open(path, access_type, recovery_mode)?);
 
         Ok(Database {
             backend,

ledger/src/snapshot_utils.rs

@@ -749,6 +749,7 @@ where
 
         bank.rc = bankrc;
         bank.operating_mode = Some(genesis_config.operating_mode);
+        bank.init_rent_collector_after_deserialize(genesis_config);
         bank.finish_init();
         Ok(bank)
     })?;

local-cluster/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-local-cluster"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -12,22 +12,22 @@ homepage = "https://solana.com/"
 itertools = "0.9.0"
 log = "0.4.8"
 rand = "0.7.0"
-solana-config-program = { path = "../programs/config", version = "1.2.19" }
-solana-core = { path = "../core", version = "1.2.19" }
-solana-client = { path = "../client", version = "1.2.19" }
-solana-download-utils = { path = "../download-utils", version = "1.2.19" }
-solana-faucet = { path = "../faucet", version = "1.2.19" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.2.19" }
-solana-genesis-programs = { path = "../genesis-programs", version = "1.2.19" }
-solana-ledger = { path = "../ledger", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-runtime = { path = "../runtime", version = "1.2.19" }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-stake-program = { path = "../programs/stake", version = "1.2.19" }
-solana-vest-program = { path = "../programs/vest", version = "1.2.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.2.19" }
+solana-config-program = { path = "../programs/config", version = "1.2.24" }
+solana-core = { path = "../core", version = "1.2.24" }
+solana-client = { path = "../client", version = "1.2.24" }
+solana-download-utils = { path = "../download-utils", version = "1.2.24" }
+solana-faucet = { path = "../faucet", version = "1.2.24" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.2.24" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.2.24" }
+solana-ledger = { path = "../ledger", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-runtime = { path = "../runtime", version = "1.2.24" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-stake-program = { path = "../programs/stake", version = "1.2.24" }
+solana-vest-program = { path = "../programs/vest", version = "1.2.24" }
+solana-vote-program = { path = "../programs/vote", version = "1.2.24" }
 tempfile = "3.1.0"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.19" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.24" }
 
 [dev-dependencies]
 assert_matches = "1.3.0"

log-analyzer/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-log-analyzer"
 description = "The solana cluster network analysis tool"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,9 +14,9 @@ byte-unit = "3.1.1"
 clap = "2.33.1"
 serde = "1.0.110"
 serde_json = "1.0.53"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
 
 [[bin]]
 name = "solana-log-analyzer"

logger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-logger"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana Logger"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

measure/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-measure"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.19"
+version = "1.2.24"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -12,8 +12,8 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-metrics = { path = "../metrics", version = "1.2.19" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-metrics = { path = "../metrics", version = "1.2.24" }
 
 [target."cfg(unix)".dependencies]
 jemallocator = "0.3.2"

merkle-tree/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-merkle-tree"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana Merkle Tree"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../sdk", version = "1.2.19" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
 fast-math = "0.1"
 
 [dev-dependencies]

metrics/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-metrics"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana Metrics"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,7 +14,7 @@ gethostname = "0.2.1"
 lazy_static = "1.4.0"
 log = "0.4.8"
 reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-solana-sdk = { path = "../sdk", version = "1.2.19" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
 
 [dev-dependencies]
 rand = "0.7.0"

multinode-demo/bootstrap-validator.sh

@@ -48,6 +48,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 = --enable-rpc-transaction-history ]]; then
       args+=("$1")
       shift
+    elif [[ $1 = --enable-rpc-bigtable-ledger-storage ]]; then
+      args+=("$1")
+      shift
     elif [[ $1 = --skip-poh-verify ]]; then
       args+=("$1")
       shift

net-shaper/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-net-shaper"
 description = "The solana cluster network shaping tool"
-version = "1.2.19"
+version = "1.2.24"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,8 +13,8 @@ publish = false
 clap = "2.33.1"
 serde = "1.0.110"
 serde_json = "1.0.53"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
 rand = "0.7.0"
 
 [[bin]]

net-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-net-utils"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana Network Utilities"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -18,9 +18,9 @@ rand = "0.7.0"
 serde = "1.0.110"
 serde_derive = "1.0.103"
 socket2 = "0.3.12"
-solana-clap-utils = { path = "../clap-utils", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-version = { path = "../version", version = "1.2.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-version = { path = "../version", version = "1.2.24" }
 tokio = "0.1"
 tokio-codec = "0.1"
 

notifier/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-notifier"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana Notifier"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

perf/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-perf"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana Performance APIs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -17,11 +17,11 @@ serde = "1.0.110"
 dlopen_derive = "0.1.4"
 lazy_static = "1.4.0"
 log = "0.4.8"
-solana-sdk = { path = "../sdk", version = "1.2.19" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.19" }
-solana-budget-program = { path = "../programs/budget", version = "1.2.19" }
-solana-logger = { path = "../logger", version = "1.2.19" }
-solana-metrics = { path = "../metrics", version = "1.2.19" }
+solana-sdk = { path = "../sdk", version = "1.2.24" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.2.24" }
+solana-budget-program = { path = "../programs/budget", version = "1.2.24" }
+solana-logger = { path = "../logger", version = "1.2.24" }
+solana-metrics = { path = "../metrics", version = "1.2.24" }
 curve25519-dalek = { version = "2" }
 
 [lib]

perf/src/sigverify.rs

@@ -131,7 +131,8 @@ fn do_get_packet_offsets(
     }
 
     // read the length of Transaction.signatures (serialized with short_vec)
-    let (sig_len_untrusted, sig_size) = decode_len(&packet.data)?;
+    let (sig_len_untrusted, sig_size) =
+        decode_len(&packet.data).map_err(|_| PacketError::InvalidShortVec)?;
 
     // Using msg_start_offset which is based on sig_len_untrusted introduces uncertainty.
     // Ultimately, the actual sigverify will determine the uncertainty.
@@ -156,8 +157,8 @@ fn do_get_packet_offsets(
     }
 
     // read the length of Message.account_keys (serialized with short_vec)
-    let (pubkey_len, pubkey_len_size) =
-        decode_len(&packet.data[message_account_keys_len_offset..])?;
+    let (pubkey_len, pubkey_len_size) = decode_len(&packet.data[message_account_keys_len_offset..])
+        .map_err(|_| PacketError::InvalidShortVec)?;
 
     if (message_account_keys_len_offset + pubkey_len * size_of::<Pubkey>() + pubkey_len_size)
         > packet.meta.size

programs/bpf/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-bpf-programs"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.2.19"
+version = "1.2.24"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "README.md"
@@ -22,10 +22,10 @@ walkdir = "2"
 bincode = "1.1.4"
 byteorder = "1.3.2"
 elf = "0.0.10"
-solana-bpf-loader-program = { path = "../bpf_loader", version = "1.2.19" }
-solana-logger = { path = "../../logger", version = "1.2.19" }
-solana-runtime = { path = "../../runtime", version = "1.2.19" }
-solana-sdk = { path = "../../sdk", version = "1.2.19" }
+solana-bpf-loader-program = { path = "../bpf_loader", version = "1.2.24" }
+solana-logger = { path = "../../logger", version = "1.2.24" }
+solana-runtime = { path = "../../runtime", version = "1.2.24" }
+solana-sdk = { path = "../../sdk", version = "1.2.24" }
 solana_rbpf = "=0.1.28"
 
 [[bench]]

programs/bpf/c/src/invoke/invoke.c

@@ -30,6 +30,10 @@ extern uint64_t entrypoint(const uint8_t *input) {
     return ERROR_INVALID_ARGUMENT;
   }
 
+  uint8_t nonce1 = params.data[1];
+  uint8_t nonce2 = params.data[2];
+  uint8_t nonce3 = params.data[3];
+
   switch (params.data[0]) {
   case TEST_SUCCESS: {
     sol_log("Call system program");
@@ -81,6 +85,18 @@ extern uint64_t entrypoint(const uint8_t *input) {
                  sol_invoke(&instruction, accounts, SOL_ARRAY_SIZE(accounts)));
     }
 
+    sol_log("Test create_program_address");
+    {
+      uint8_t seed1[] = {'Y', 'o', 'u', ' ', 'p', 'a', 's', 's',
+                         ' ', 'b', 'u', 't', 't', 'e', 'r'};
+      const SolSignerSeed seeds1[] = {{seed1, SOL_ARRAY_SIZE(seed1)},
+                                      {&nonce1, 1}};
+      SolPubkey address;
+      sol_assert(SUCCESS == sol_create_program_address(seeds1, SOL_ARRAY_SIZE(seeds1),
+                                                 params.program_id, &address));
+      sol_assert(SolPubkey_same(&address, accounts[DERIVED_KEY1_INDEX].key));
+    }
+
     sol_log("Test derived signers");
     {
       sol_assert(!accounts[DERIVED_KEY1_INDEX].is_signer);
@@ -92,19 +108,15 @@ extern uint64_t entrypoint(const uint8_t *input) {
           {accounts[DERIVED_KEY1_INDEX].key, true, true},
           {accounts[DERIVED_KEY2_INDEX].key, true, false},
           {accounts[DERIVED_KEY3_INDEX].key, false, false}};
-      uint8_t data[] = {TEST_DERIVED_SIGNERS};
+      uint8_t data[] = {TEST_DERIVED_SIGNERS, nonce2, nonce3};
       const SolInstruction instruction = {accounts[INVOKED_PROGRAM_INDEX].key,
                                           arguments, SOL_ARRAY_SIZE(arguments),
                                           data, SOL_ARRAY_SIZE(data)};
       uint8_t seed1[] = {'Y', 'o', 'u', ' ', 'p', 'a', 's', 's',
                          ' ', 'b', 'u', 't', 't', 'e', 'r'};
-      uint8_t seed2[] = {'L', 'i', 'l', '\''};
-      uint8_t seed3[] = {'B', 'i', 't', 's'};
-      const SolSignerSeed seeds1[] = {{seed1, SOL_ARRAY_SIZE(seed1)}};
-      const SolSignerSeed seeds2[] = {{seed2, SOL_ARRAY_SIZE(seed2)},
-                                      {seed3, SOL_ARRAY_SIZE(seed3)}};
-      const SolSignerSeeds signers_seeds[] = {{seeds1, SOL_ARRAY_SIZE(seeds1)},
-                                              {seeds2, SOL_ARRAY_SIZE(seeds2)}};
+      const SolSignerSeed seeds1[] = {{seed1, SOL_ARRAY_SIZE(seed1)},
+                                      {&nonce1, 1}};
+      const SolSignerSeeds signers_seeds[] = {{seeds1, SOL_ARRAY_SIZE(seeds1)}};
       sol_assert(SUCCESS == sol_invoke_signed(&instruction, accounts,
                                               SOL_ARRAY_SIZE(accounts),
                                               signers_seeds,

programs/bpf/c/src/invoked/invoked.c

@@ -36,7 +36,7 @@ extern uint64_t entrypoint(const uint8_t *input) {
     sol_assert(accounts[ARGUMENT_INDEX].data_len == 100);
     sol_assert(accounts[ARGUMENT_INDEX].is_signer);
     sol_assert(accounts[ARGUMENT_INDEX].is_writable);
-    sol_assert(accounts[ARGUMENT_INDEX].rent_epoch == 1);
+    sol_assert(accounts[ARGUMENT_INDEX].rent_epoch == 0);
     sol_assert(!accounts[ARGUMENT_INDEX].executable);
     for (int i = 0; i < accounts[ARGUMENT_INDEX].data_len; i++) {
       sol_assert(accounts[ARGUMENT_INDEX].data[i] == i);
@@ -48,7 +48,7 @@ extern uint64_t entrypoint(const uint8_t *input) {
     sol_assert(accounts[INVOKED_ARGUMENT_INDEX].data_len == 10);
     sol_assert(accounts[INVOKED_ARGUMENT_INDEX].is_signer);
     sol_assert(accounts[INVOKED_ARGUMENT_INDEX].is_writable);
-    sol_assert(accounts[INVOKED_ARGUMENT_INDEX].rent_epoch == 1);
+    sol_assert(accounts[INVOKED_ARGUMENT_INDEX].rent_epoch == 0);
     sol_assert(!accounts[INVOKED_ARGUMENT_INDEX].executable);
 
     sol_assert(
@@ -57,7 +57,7 @@ extern uint64_t entrypoint(const uint8_t *input) {
                                   &bpf_loader_id));
     sol_assert(!accounts[INVOKED_PROGRAM_INDEX].is_signer);
     sol_assert(!accounts[INVOKED_PROGRAM_INDEX].is_writable);
-    sol_assert(accounts[INVOKED_PROGRAM_INDEX].rent_epoch == 1);
+    sol_assert(accounts[INVOKED_PROGRAM_INDEX].rent_epoch == 0);
     sol_assert(accounts[INVOKED_PROGRAM_INDEX].executable);
 
     sol_assert(SolPubkey_same(accounts[INVOKED_PROGRAM_INDEX].key,
@@ -92,6 +92,9 @@ extern uint64_t entrypoint(const uint8_t *input) {
     sol_assert(!accounts[DERIVED_KEY2_INDEX].is_signer);
     sol_assert(!accounts[DERIVED_KEY2_INDEX].is_signer);
 
+    uint8_t nonce2 = params.data[1];
+    uint8_t nonce3 = params.data[2];
+
     SolAccountMeta arguments[] = {
         {accounts[DERIVED_KEY1_INDEX].key, true, false},
         {accounts[DERIVED_KEY2_INDEX].key, true, true},
@@ -103,9 +106,11 @@ extern uint64_t entrypoint(const uint8_t *input) {
     uint8_t seed1[] = {'L', 'i', 'l', '\''};
     uint8_t seed2[] = {'B', 'i', 't', 's'};
     const SolSignerSeed seeds1[] = {{seed1, SOL_ARRAY_SIZE(seed1)},
-                                    {seed2, SOL_ARRAY_SIZE(seed2)}};
+                                    {seed2, SOL_ARRAY_SIZE(seed2)},
+                                    {&nonce2, 1}};
     const SolSignerSeed seeds2[] = {
-        {(uint8_t *)accounts[DERIVED_KEY2_INDEX].key, SIZE_PUBKEY}};
+        {(uint8_t *)accounts[DERIVED_KEY2_INDEX].key, SIZE_PUBKEY},
+        {&nonce3, 1}};
     const SolSignerSeeds signers_seeds[] = {{seeds1, SOL_ARRAY_SIZE(seeds1)},
                                             {seeds2, SOL_ARRAY_SIZE(seeds2)}};
 

programs/bpf/rust/128bit/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "solana-bpf-rust-128bit"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,8 +12,8 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../../../../sdk/", version = "1.2.19", default-features = false }
-solana-bpf-rust-128bit-dep = { path = "../128bit_dep", version = "1.2.19" }
+solana-sdk = { path = "../../../../sdk/", version = "1.2.24", default-features = false }
+solana-bpf-rust-128bit-dep = { path = "../128bit_dep", version = "1.2.24" }
 
 [features]
 program = ["solana-sdk/program"]

programs/bpf/rust/128bit_dep/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "solana-bpf-rust-128bit-dep"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,7 +12,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../../../../sdk/", version = "1.2.19", default-features = false }
+solana-sdk = { path = "../../../../sdk/", version = "1.2.24", default-features = false }
 
 [features]
 program = ["solana-sdk/program"]

programs/bpf/rust/alloc/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "solana-bpf-rust-alloc"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,7 +12,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../../../../sdk/", version = "1.2.19", default-features = false }
+solana-sdk = { path = "../../../../sdk/", version = "1.2.24", default-features = false }
 
 [features]
 program = ["solana-sdk/program"]

programs/bpf/rust/dep_crate/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "solana-bpf-rust-dep-crate"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,7 +13,7 @@ edition = "2018"
 
 [dependencies]
 byteorder = { version = "1", default-features = false }
-solana-sdk = { path = "../../../../sdk/", version = "1.2.19", default-features = false }
+solana-sdk = { path = "../../../../sdk/", version = "1.2.24", default-features = false }
 
 [features]
 program = ["solana-sdk/program"]

programs/bpf/rust/dup_accounts/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "solana-bpf-rust-dup-accounts"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,7 +12,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../../../../sdk/", version = "1.2.19", default-features = false }
+solana-sdk = { path = "../../../../sdk/", version = "1.2.24", default-features = false }
 
 [features]
 program = ["solana-sdk/program"]

programs/bpf/rust/error_handling/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "solana-bpf-rust-error-handling"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,7 +14,7 @@ edition = "2018"
 [dependencies]
 num-derive = "0.2"
 num-traits = "0.2"
-solana-sdk = { path = "../../../../sdk/", version = "1.2.19", default-features = false }
+solana-sdk = { path = "../../../../sdk/", version = "1.2.24", default-features = false }
 thiserror = "1.0"
 
 [features]

programs/bpf/rust/external_spend/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "solana-bpf-rust-external-spend"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,7 +12,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../../../../sdk/", version = "1.2.19", default-features = false }
+solana-sdk = { path = "../../../../sdk/", version = "1.2.24", default-features = false }
 
 [features]
 program = ["solana-sdk/program"]

programs/bpf/rust/invoke/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "solana-bpf-rust-invoke"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,7 +13,7 @@ edition = "2018"
 
 [dependencies]
 solana-bpf-rust-invoked = { path = "../invoked"}
-solana-sdk = { path = "../../../../sdk/", version = "1.2.19", default-features = false }
+solana-sdk = { path = "../../../../sdk/", version = "1.2.24", default-features = false }
 
 [features]
 program = ["solana-sdk/program"]

programs/bpf/rust/invoke/src/lib.rs

@@ -10,7 +10,7 @@ use solana_sdk::{
     entrypoint,
     entrypoint::ProgramResult,
     info,
-    program::{invoke, invoke_signed},
+    program::{create_program_address, invoke, invoke_signed},
     program_error::ProgramError,
     pubkey::Pubkey,
     system_instruction,
@@ -34,12 +34,16 @@ const FROM_INDEX: usize = 10;
 
 entrypoint!(process_instruction);
 fn process_instruction(
-    _program_id: &Pubkey,
+    program_id: &Pubkey,
     accounts: &[AccountInfo],
     instruction_data: &[u8],
 ) -> ProgramResult {
     info!("invoke Rust program");
 
+    let nonce1 = instruction_data[1];
+    let nonce2 = instruction_data[2];
+    let nonce3 = instruction_data[3];
+
     match instruction_data[0] {
         TEST_SUCCESS => {
             info!("Call system program");
@@ -91,6 +95,12 @@ fn process_instruction(
                 );
             }
 
+            info!("Test create_program_address");
+            {
+                let address = create_program_address(&[b"You pass butter", &[nonce1]], program_id)?;
+                assert_eq!(&address, accounts[DERIVED_KEY1_INDEX].key);
+            }
+
             info!("Test derived signers");
             {
                 assert!(!accounts[DERIVED_KEY1_INDEX].is_signer);
@@ -105,12 +115,12 @@ fn process_instruction(
                         (accounts[DERIVED_KEY2_INDEX].key, true, false),
                         (accounts[DERIVED_KEY3_INDEX].key, false, false),
                     ],
-                    vec![TEST_DERIVED_SIGNERS],
+                    vec![TEST_DERIVED_SIGNERS, nonce2, nonce3],
                 );
                 invoke_signed(
                     &invoked_instruction,
                     accounts,
-                    &[&[b"You pass butter"], &[b"Lil'", b"Bits"]],
+                    &[&[b"You pass butter", &[nonce1]]],
                 )?;
             }
 

programs/bpf/rust/invoked/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "solana-bpf-rust-invoked"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,7 +12,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../../../../sdk/", version = "1.2.19", default-features = false }
+solana-sdk = { path = "../../../../sdk/", version = "1.2.24", default-features = false }
 
 [features]
 program = ["solana-sdk/program"]

programs/bpf/rust/invoked/src/lib.rs

@@ -42,7 +42,7 @@ fn process_instruction(
             assert_eq!(accounts[ARGUMENT_INDEX].data_len(), 100);
             assert!(accounts[ARGUMENT_INDEX].is_signer);
             assert!(accounts[ARGUMENT_INDEX].is_writable);
-            assert_eq!(accounts[ARGUMENT_INDEX].rent_epoch, 1);
+            assert_eq!(accounts[ARGUMENT_INDEX].rent_epoch, 0);
             assert!(!accounts[ARGUMENT_INDEX].executable);
             {
                 let data = accounts[ARGUMENT_INDEX].try_borrow_data()?;
@@ -59,14 +59,14 @@ fn process_instruction(
             assert_eq!(accounts[INVOKED_ARGUMENT_INDEX].data_len(), 10);
             assert!(accounts[INVOKED_ARGUMENT_INDEX].is_signer);
             assert!(accounts[INVOKED_ARGUMENT_INDEX].is_writable);
-            assert_eq!(accounts[INVOKED_ARGUMENT_INDEX].rent_epoch, 1);
+            assert_eq!(accounts[INVOKED_ARGUMENT_INDEX].rent_epoch, 0);
             assert!(!accounts[INVOKED_ARGUMENT_INDEX].executable);
 
             assert_eq!(accounts[INVOKED_PROGRAM_INDEX].key, program_id);
             assert_eq!(accounts[INVOKED_PROGRAM_INDEX].owner, &bpf_loader::id());
             assert!(!accounts[INVOKED_PROGRAM_INDEX].is_signer);
             assert!(!accounts[INVOKED_PROGRAM_INDEX].is_writable);
-            assert_eq!(accounts[INVOKED_PROGRAM_INDEX].rent_epoch, 1);
+            assert_eq!(accounts[INVOKED_PROGRAM_INDEX].rent_epoch, 0);
             assert!(accounts[INVOKED_PROGRAM_INDEX].executable);
 
             assert_eq!(
@@ -120,6 +120,8 @@ fn process_instruction(
             assert!(!accounts[DERIVED_KEY2_INDEX].is_signer);
             assert!(!accounts[DERIVED_KEY3_INDEX].is_signer);
 
+            let nonce2 = instruction_data[1];
+            let nonce3 = instruction_data[2];
             let invoked_instruction = create_instruction(
                 *accounts[INVOKED_PROGRAM_INDEX].key,
                 &[
@@ -133,8 +135,8 @@ fn process_instruction(
                 &invoked_instruction,
                 accounts,
                 &[
-                    &[b"Lil'", b"Bits"],
-                    &[accounts[DERIVED_KEY2_INDEX].key.as_ref()],
+                    &[b"Lil'", b"Bits", &[nonce2]],
+                    &[accounts[DERIVED_KEY2_INDEX].key.as_ref(), &[nonce3]],
                 ],
             )?;
         }

programs/bpf/rust/iter/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "solana-bpf-rust-iter"
-version = "1.2.19"
+version = "1.2.24"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,7 +12,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../../../../sdk/", version = "1.2.19", default-features = false }
+solana-sdk = { path = "../../../../sdk/", version = "1.2.24", default-features = false }
 
 [features]
 program = ["solana-sdk/program"]