gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

caches vote-state de-serialized from vote accounts (#13795)

Browse Source 
Gossip and other places repeatedly de-serialize vote-state stored in
vote accounts. Ideally the first de-serialization should cache the
result.

This commit adds new VoteAccount type which lazily de-serializes
VoteState from Account data and caches the result internally.

Serialize and Deserialize traits are manually implemented to match
existing code. So, despite changes to frozen_abi, this commit should be
backward compatible.
This commit is contained in:
behzad nouri
2020-11-30 17:18:33 +00:00
committed by GitHub
parent 6203d1c94c
commit e1793e5a13
18 changed files with 433 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
ledger/src/blockstore.rs
View File

@ -24,9 +24,11 @@ use rocksdb::DBRawIterator;
use solana_measure::measure::Measure;
use solana_metrics::{datapoint_debug, datapoint_error};
use solana_rayon_threadlimit::get_thread_count;
use solana_runtime::hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE};
use solana_runtime::{
hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE},
vote_account::ArcVoteAccount,
};
use solana_sdk::{
account::Account,
clock::{Slot, UnixTimestamp, DEFAULT_TICKS_PER_SECOND, MS_PER_TICK},
genesis_config::GenesisConfig,
hash::Hash,
@ -1623,7 +1625,7 @@ impl Blockstore {
&self,
slot: Slot,
slot_duration: Duration,
stakes: &HashMap<Pubkey, (u64, Account)>,
stakes: &HashMap<Pubkey, (u64, ArcVoteAccount)>,
) -> Result<()> {
if !self.is_root(slot) {
return Err(BlockstoreError::SlotNotRooted);
@ -5817,7 +5819,7 @@ pub mod tests {
// Build epoch vote_accounts HashMap to test stake-weighted block time
for (i, keypair) in vote_keypairs.iter().enumerate() {
stakes.insert(keypair.pubkey(), (1 + i as u64, Account::default()));
stakes.insert(keypair.pubkey(), (1 + i as u64, ArcVoteAccount::default()));
}
for slot in &[1, 2, 3, 8] {
blockstore
@ -5876,7 +5878,7 @@ pub mod tests {
// Build epoch vote_accounts HashMap to test stake-weighted block time
let mut stakes = HashMap::new();
for (i, keypair) in vote_keypairs.iter().enumerate() {
stakes.insert(keypair.pubkey(), (1 + i as u64, Account::default()));
stakes.insert(keypair.pubkey(), (1 + i as u64, ArcVoteAccount::default()));
}
let slot_duration = Duration::from_millis(400);
for slot in &[1, 2, 3, 8] {

48
ledger/src/blockstore_processor.rs
View File

@ -24,10 +24,10 @@ use solana_runtime::{
commitment::VOTE_THRESHOLD_SIZE,
transaction_batch::TransactionBatch,
transaction_utils::OrderedIterator,
vote_account::ArcVoteAccount,
vote_sender_types::ReplayVoteSender,
};
use solana_sdk::{
account::Account,
clock::{Slot, MAX_PROCESSING_AGE},
genesis_config::GenesisConfig,
hash::Hash,
@ -36,7 +36,6 @@ use solana_sdk::{
timing::duration_as_ms,
transaction::{Result, Transaction, TransactionError},
};
use solana_vote_program::vote_state::VoteState;
use std::{
cell::RefCell,
collections::{HashMap, HashSet},
@ -868,8 +867,11 @@ fn load_frozen_forks(
// for newer cluster confirmed roots
let new_root_bank = {
if *root == max_root {
supermajority_root_from_vote_accounts(bank.slot(), bank.total_epoch_stake(), bank.vote_accounts()
.into_iter()).and_then(|supermajority_root| {
supermajority_root_from_vote_accounts(
bank.slot(),
bank.total_epoch_stake(),
bank.vote_accounts(),
).and_then(|supermajority_root| {
if supermajority_root > *root {
// If there's a cluster confirmed root greater than our last
// replayed root, then beccause the cluster confirmed root should
@ -960,30 +962,28 @@ fn supermajority_root(roots: &[(Slot, u64)], total_epoch_stake: u64) -> Option<S
fn supermajority_root_from_vote_accounts<I>(
bank_slot: Slot,
total_epoch_stake: u64,
vote_accounts_iter: I,
vote_accounts: I,
) -> Option<Slot>
where
I: Iterator<Item = (Pubkey, (u64, Account))>,
I: IntoIterator<Item = (Pubkey, (u64, ArcVoteAccount))>,
{
let mut roots_stakes: Vec<(Slot, u64)> = vote_accounts_iter
let mut roots_stakes: Vec<(Slot, u64)> = vote_accounts
.into_iter()
.filter_map(|(key, (stake, account))| {
if stake == 0 {
return None;
}
let vote_state = VoteState::from(&account);
if vote_state.is_none() {
warn!(
"Unable to get vote_state from account {} in bank: {}",
key, bank_slot
);
return None;
match account.vote_state().as_ref() {
Err(_) => {
warn!(
"Unable to get vote_state from account {} in bank: {}",
key, bank_slot
);
None
}
Ok(vote_state) => vote_state.root_slot.map(|root_slot| (root_slot, stake)),
}
vote_state
.unwrap()
.root_slot
.map(|root_slot| (root_slot, stake))
})
.collect();
@ -1112,6 +1112,7 @@ pub mod tests {
self, create_genesis_config_with_vote_accounts, ValidatorVoteKeypairs,
};
use solana_sdk::{
account::Account,
epoch_schedule::EpochSchedule,
hash::Hash,
pubkey::Pubkey,
@ -1122,7 +1123,7 @@ pub mod tests {
};
use solana_vote_program::{
self,
vote_state::{VoteStateVersions, MAX_LOCKOUT_HISTORY},
vote_state::{VoteState, VoteStateVersions, MAX_LOCKOUT_HISTORY},
vote_transaction,
};
use std::{collections::BTreeSet, sync::RwLock};
@ -3146,7 +3147,7 @@ pub mod tests {
#[test]
fn test_supermajority_root_from_vote_accounts() {
let convert_to_vote_accounts =
|roots_stakes: Vec<(Slot, u64)>| -> Vec<(Pubkey, (u64, Account))> {
|roots_stakes: Vec<(Slot, u64)>| -> Vec<(Pubkey, (u64, ArcVoteAccount))> {
roots_stakes
.into_iter()
.map(|(root, stake)| {
@ -3156,7 +3157,10 @@ pub mod tests {
Account::new(1, VoteState::size_of(), &solana_vote_program::id());
let versioned = VoteStateVersions::Current(Box::new(vote_state));
VoteState::serialize(&versioned, &mut vote_account.data).unwrap();
(solana_sdk::pubkey::new_rand(), (stake, vote_account))
(
solana_sdk::pubkey::new_rand(),
(stake, ArcVoteAccount::from(vote_account)),
)
})
.collect_vec()
};

82
ledger/src/staking_utils.rs
View File

@ -1,10 +1,8 @@
use solana_runtime::bank::Bank;
use solana_runtime::{bank::Bank, vote_account::ArcVoteAccount};
use solana_sdk::{
account::Account,
clock::{Epoch, Slot},
pubkey::Pubkey,
};
use solana_vote_program::vote_state::VoteState;
use std::{borrow::Borrow, collections::HashMap};
/// Looks through vote accounts, and finds the latest slot that has achieved
@ -28,48 +26,42 @@ pub fn vote_account_stakes(bank: &Bank) -> HashMap<Pubkey, u64> {
/// Collect the staked nodes, as named by staked vote accounts from the given bank
pub fn staked_nodes(bank: &Bank) -> HashMap<Pubkey, u64> {
to_staked_nodes(to_vote_states(bank.vote_accounts().into_iter()))
to_staked_nodes(bank.vote_accounts())
}
/// At the specified epoch, collect the delegate account balance and vote states for delegates
/// that have non-zero balance in any of their managed staking accounts
pub fn staked_nodes_at_epoch(bank: &Bank, epoch: Epoch) -> Option<HashMap<Pubkey, u64>> {
bank.epoch_vote_accounts(epoch)
.map(|vote_accounts| to_staked_nodes(to_vote_states(vote_accounts.iter())))
bank.epoch_vote_accounts(epoch).map(to_staked_nodes)
}
// input (vote_pubkey, (stake, vote_account)) => (stake, vote_state)
fn to_vote_states(
node_staked_accounts: impl Iterator<Item = (impl Borrow<Pubkey>, impl Borrow<(u64, Account)>)>,
) -> impl Iterator<Item = (u64, VoteState)> {
node_staked_accounts.filter_map(|(_, stake_account)| {
VoteState::deserialize(&stake_account.borrow().1.data)
.ok()
.map(|vote_state| (stake_account.borrow().0, vote_state))
})
}
// (stake, vote_state) => (node, stake)
fn to_staked_nodes(
node_staked_accounts: impl Iterator<Item = (u64, VoteState)>,
) -> HashMap<Pubkey, u64> {
let mut map: HashMap<Pubkey, u64> = HashMap::new();
node_staked_accounts.for_each(|(stake, state)| {
map.entry(state.node_pubkey)
.and_modify(|s| *s += stake)
.or_insert(stake);
});
map
fn to_staked_nodes<I, K, V>(
vote_accounts: I,
) -> HashMap<Pubkey /*VoteState.node_pubkey*/, u64 /*stake*/>
where
I: IntoIterator<Item = (K /*vote pubkey*/, V)>,
V: Borrow<(u64 /*stake*/, ArcVoteAccount)>,
{
let mut out: HashMap<Pubkey, u64> = HashMap::new();
for (_ /*vote pubkey*/, stake_vote_account) in vote_accounts {
let (stake, vote_account) = stake_vote_account.borrow();
if let Ok(vote_state) = vote_account.vote_state().as_ref() {
out.entry(vote_state.node_pubkey)
.and_modify(|s| *s += *stake)
.or_insert(*stake);
}
}
out
}
fn epoch_stakes_and_lockouts(bank: &Bank, epoch: Epoch) -> Vec<(u64, Option<u64>)> {
let node_staked_accounts = bank
.epoch_vote_accounts(epoch)
bank.epoch_vote_accounts(epoch)
.expect("Bank state for epoch is missing")
.iter();
to_vote_states(node_staked_accounts)
.map(|(stake, states)| (stake, states.root_slot))
.iter()
.filter_map(|(_ /*vote pubkey*/, (stake, vote_account))| {
let root_slot = vote_account.vote_state().as_ref().ok()?.root_slot;
Some((*stake, root_slot))
})
.collect()
}
@ -103,8 +95,9 @@ pub(crate) mod tests {
use crate::genesis_utils::{
bootstrap_validator_stake_lamports, create_genesis_config, GenesisConfigInfo,
};
use rand::Rng;
use solana_sdk::{
account::from_account,
account::{from_account, Account},
clock::Clock,
instruction::Instruction,
pubkey::Pubkey,
@ -117,7 +110,10 @@ pub(crate) mod tests {
stake_instruction,
stake_state::{Authorized, Delegation, Lockup, Stake},
};
use solana_vote_program::{vote_instruction, vote_state::VoteInit};
use solana_vote_program::{
vote_instruction,
vote_state::{VoteInit, VoteState, VoteStateVersions},
};
use std::sync::Arc;
fn new_from_parent(parent: &Arc<Bank>, slot: Slot) -> Bank {
@ -340,8 +336,18 @@ pub(crate) mod tests {
&Clock::default(),
),
));
let result = to_staked_nodes(stakes.into_iter());
let mut rng = rand::thread_rng();
let vote_accounts = stakes.into_iter().map(|(stake, vote_state)| {
let account = Account::new_data(
rng.gen(), // lamports
&VoteStateVersions::Current(Box::new(vote_state)),
&Pubkey::new_unique(), // owner
)
.unwrap();
let vote_pubkey = Pubkey::new_unique();
(vote_pubkey, (stake, ArcVoteAccount::from(account)))
});
let result = to_staked_nodes(vote_accounts);
assert_eq!(result.len(), 2);
assert_eq!(result[&node1], 3);
assert_eq!(result[&node2], 5);