From d4ddb6265bf57a40668bab34d2578acdb021da82 Mon Sep 17 00:00:00 2001 From: carllin Date: Wed, 25 Mar 2020 20:43:48 -0700 Subject: [PATCH] Convert Banks (#9033) * Store and compute needed state in EpochStakes struct Co-authored-by: Carl --- ledger/src/snapshot_utils.rs | 10 +- runtime/src/bank/bank_1_0.rs | 220 +++++++++++++++++++++++++++ runtime/src/{bank.rs => bank/mod.rs} | 73 ++++++++- runtime/src/epoch_stakes.rs | 208 +++++++++++++++++++++++++ runtime/src/lib.rs | 1 + 5 files changed, 506 insertions(+), 6 deletions(-) create mode 100644 runtime/src/bank/bank_1_0.rs rename runtime/src/{bank.rs => bank/mod.rs} (98%) create mode 100644 runtime/src/epoch_stakes.rs diff --git a/ledger/src/snapshot_utils.rs b/ledger/src/snapshot_utils.rs index 5564feaf4e..c4b263af31 100644 --- a/ledger/src/snapshot_utils.rs +++ b/ledger/src/snapshot_utils.rs @@ -8,7 +8,7 @@ use solana_measure::measure::Measure; use solana_runtime::{ accounts_db::{SnapshotStorage, SnapshotStorages}, bank::{ - self, deserialize_from_snapshot, Bank, BankRcSerialize, BankSlotDelta, + self, bank_1_0::Bank1_0, deserialize_from_snapshot, Bank, BankRcSerialize, BankSlotDelta, MAX_SNAPSHOT_DATA_FILE_SIZE, }, }; @@ -30,7 +30,8 @@ pub const TAR_SNAPSHOTS_DIR: &str = "snapshots"; pub const TAR_ACCOUNTS_DIR: &str = "accounts"; pub const TAR_VERSION_FILE: &str = "version"; -pub const SNAPSHOT_VERSION: &str = "1.0.0"; +pub const SNAPSHOT_VERSION_1_0: &str = "1.0.0"; +pub const SNAPSHOT_VERSION: &str = "1.1.0"; #[derive(PartialEq, Ord, Eq, Debug)] pub struct SlotSnapshotPaths { @@ -593,6 +594,10 @@ where MAX_SNAPSHOT_DATA_FILE_SIZE, |stream| { let mut bank: Bank = match snapshot_version { + SNAPSHOT_VERSION_1_0 => { + let bank_1_0: Bank1_0 = deserialize_from_snapshot(stream.by_ref())?; + bank_1_0.convert_to_current() + } SNAPSHOT_VERSION => deserialize_from_snapshot(stream.by_ref())?, _ => { return Err(get_io_error(&format!( @@ -602,6 +607,7 @@ where } }; info!("Rebuilding accounts..."); + let rc = bank::BankRc::from_stream( account_paths, bank.slot(), diff --git a/runtime/src/bank/bank_1_0.rs b/runtime/src/bank/bank_1_0.rs new file mode 100644 index 0000000000..62370467ee --- /dev/null +++ b/runtime/src/bank/bank_1_0.rs @@ -0,0 +1,220 @@ +//! The `bank` module tracks client accounts and the progress of on-chain +//! programs. It offers a high-level API that signs transactions +//! on behalf of the caller, and a low-level API for when they have +//! already been signed and verified. +use crate::{ + bank::{Bank, BankRc, EnteredEpochCallback, StatusCacheRc}, + blockhash_queue::BlockhashQueue, + epoch_stakes::EpochStakes, + message_processor::MessageProcessor, + rent_collector::RentCollector, + serde_utils::{ + deserialize_atomicbool, deserialize_atomicu64, serialize_atomicbool, serialize_atomicu64, + }, + stakes::Stakes, + storage_utils::StorageAccounts, +}; +use serde::{Deserialize, Serialize}; +use solana_sdk::{ + clock::{Epoch, Slot, UnixTimestamp}, + epoch_schedule::EpochSchedule, + fee_calculator::{FeeCalculator, FeeRateGovernor}, + hard_forks::HardForks, + hash::Hash, + inflation::Inflation, + pubkey::Pubkey, +}; +use std::{ + collections::HashMap, + sync::atomic::{AtomicBool, AtomicU64}, + sync::{Arc, RwLock}, +}; + +/// Manager for the state of all accounts and programs after processing its entries. +#[derive(Deserialize, Serialize)] +pub struct Bank1_0 { + /// References to accounts, parent and signature status + #[serde(skip)] + pub rc: BankRc, + + #[serde(skip)] + pub src: StatusCacheRc, + + /// FIFO queue of `recent_blockhash` items + pub blockhash_queue: RwLock, + + /// The set of parents including this bank + pub ancestors: HashMap, + + /// Hash of this Bank's state. Only meaningful after freezing. + pub hash: RwLock, + + /// Hash of this Bank's parent's state + pub parent_hash: Hash, + + /// parent's slot + pub parent_slot: Slot, + + /// slots to hard fork at + pub hard_forks: Arc>, + + /// The number of transactions processed without error + #[serde(serialize_with = "serialize_atomicu64")] + #[serde(deserialize_with = "deserialize_atomicu64")] + pub transaction_count: AtomicU64, + + /// Bank tick height + #[serde(serialize_with = "serialize_atomicu64")] + #[serde(deserialize_with = "deserialize_atomicu64")] + pub tick_height: AtomicU64, + + /// The number of signatures from valid transactions in this slot + #[serde(serialize_with = "serialize_atomicu64")] + #[serde(deserialize_with = "deserialize_atomicu64")] + pub signature_count: AtomicU64, + + /// Total capitalization, used to calculate inflation + #[serde(serialize_with = "serialize_atomicu64")] + #[serde(deserialize_with = "deserialize_atomicu64")] + pub capitalization: AtomicU64, + + // Bank max_tick_height + pub max_tick_height: u64, + + /// The number of hashes in each tick. None value means hashing is disabled. + pub hashes_per_tick: Option, + + /// The number of ticks in each slot. + pub ticks_per_slot: u64, + + /// length of a slot in ns + pub ns_per_slot: u128, + + /// genesis time, used for computed clock + pub genesis_creation_time: UnixTimestamp, + + /// The number of slots per year, used for inflation + pub slots_per_year: f64, + + /// The number of slots per Storage segment + pub slots_per_segment: u64, + + /// Bank slot (i.e. block) + pub slot: Slot, + + /// Bank epoch + pub epoch: Epoch, + + /// Bank block_height + pub block_height: u64, + + /// The pubkey to send transactions fees to. + pub collector_id: Pubkey, + + /// Fees that have been collected + #[serde(serialize_with = "serialize_atomicu64")] + #[serde(deserialize_with = "deserialize_atomicu64")] + pub collector_fees: AtomicU64, + + /// Latest transaction fees for transactions processed by this bank + pub fee_calculator: FeeCalculator, + + /// Track cluster signature throughput and adjust fee rate + pub fee_rate_governor: FeeRateGovernor, + + /// Rent that have been collected + #[serde(serialize_with = "serialize_atomicu64")] + #[serde(deserialize_with = "deserialize_atomicu64")] + pub collected_rent: AtomicU64, + + /// latest rent collector, knows the epoch + pub rent_collector: RentCollector, + + /// initialized from genesis + pub epoch_schedule: EpochSchedule, + + /// inflation specs + pub inflation: Arc>, + + /// cache of vote_account and stake_account state for this fork + pub stakes: RwLock, + + /// cache of validator and archiver storage accounts for this fork + pub storage_accounts: RwLock, + + /// staked nodes on epoch boundaries, saved off when a bank.slot() is at + /// a leader schedule calculation boundary + pub epoch_stakes: HashMap, + + /// A boolean reflecting whether any entries were recorded into the PoH + /// stream for the slot == self.slot + #[serde(serialize_with = "serialize_atomicbool")] + #[serde(deserialize_with = "deserialize_atomicbool")] + pub is_delta: AtomicBool, + + /// The Message processor + pub message_processor: MessageProcessor, + + /// Callback to be notified when a bank enters a new Epoch + /// (used to adjust cluster features over time) + #[serde(skip)] + pub entered_epoch_callback: Arc>>, + + /// Last time when the cluster info vote listener has synced with this bank + #[serde(skip)] + pub last_vote_sync: AtomicU64, + + /// Rewards that were paid out immediately after this bank was created + #[serde(skip)] + pub rewards: Option>, +} + +impl Bank1_0 { + pub fn convert_to_current(self) -> Bank { + let old_epoch_stakes = self.epoch_stakes; + let epoch_stakes = old_epoch_stakes + .iter() + .map(|(epoch, stakes)| (*epoch, EpochStakes::new(&stakes, *epoch))) + .collect(); + Bank { + rc: self.rc, + src: self.src, + blockhash_queue: self.blockhash_queue, + ancestors: self.ancestors, + hash: self.hash, + parent_hash: self.parent_hash, + parent_slot: self.parent_slot, + hard_forks: self.hard_forks, + transaction_count: self.transaction_count, + tick_height: self.tick_height, + signature_count: self.signature_count, + capitalization: self.capitalization, + max_tick_height: self.max_tick_height, + hashes_per_tick: self.hashes_per_tick, + ticks_per_slot: self.ticks_per_slot, + ns_per_slot: self.ns_per_slot, + genesis_creation_time: self.genesis_creation_time, + slots_per_year: self.slots_per_year, + slots_per_segment: self.slots_per_segment, + slot: self.slot, + epoch: self.epoch, + block_height: self.block_height, + collector_id: self.collector_id, + collector_fees: self.collector_fees, + fee_calculator: self.fee_calculator, + fee_rate_governor: self.fee_rate_governor, + collected_rent: self.collected_rent, + rent_collector: self.rent_collector, + epoch_schedule: self.epoch_schedule, + inflation: self.inflation, + stakes: self.stakes, + storage_accounts: self.storage_accounts, + epoch_stakes, + is_delta: self.is_delta, + message_processor: self.message_processor, + entered_epoch_callback: self.entered_epoch_callback, + last_vote_sync: self.last_vote_sync, + rewards: self.rewards, + } + } +} diff --git a/runtime/src/bank.rs b/runtime/src/bank/mod.rs similarity index 98% rename from runtime/src/bank.rs rename to runtime/src/bank/mod.rs index f61fd3738b..1e231c0f31 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank/mod.rs @@ -6,6 +6,7 @@ use crate::{ accounts::{Accounts, TransactionAccounts, TransactionLoadResult, TransactionLoaders}, accounts_db::{AccountsDBSerialize, ErrorCounters, SnapshotStorage, SnapshotStorages}, blockhash_queue::BlockhashQueue, + epoch_stakes::{EpochStakes, NodeVoteAccounts}, message_processor::{MessageProcessor, ProcessInstruction}, nonce_utils, rent_collector::RentCollector, @@ -60,6 +61,8 @@ use std::{ sync::{Arc, RwLock, RwLockReadGuard}, }; +pub mod bank_1_0; + pub const SECONDS_PER_YEAR: f64 = 365.25 * 24.0 * 60.0 * 60.0; pub const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB @@ -323,7 +326,7 @@ pub struct Bank { /// staked nodes on epoch boundaries, saved off when a bank.slot() is at /// a leader schedule calculation boundary - epoch_stakes: HashMap, + epoch_stakes: HashMap, /// A boolean reflecting whether any entries were recorded into the PoH /// stream for the slot == self.slot @@ -380,7 +383,8 @@ impl Bank { { let stakes = bank.stakes.read().unwrap(); for epoch in 0..=bank.get_leader_schedule_epoch(bank.slot) { - bank.epoch_stakes.insert(epoch, stakes.clone()); + bank.epoch_stakes + .insert(epoch, EpochStakes::new(&stakes, epoch)); } bank.update_stake_history(None); } @@ -592,8 +596,24 @@ impl Bank { epoch >= leader_schedule_epoch.saturating_sub(MAX_LEADER_SCHEDULE_STAKES) }); + let vote_stakes: HashMap<_, _> = self + .stakes + .read() + .unwrap() + .vote_accounts() + .iter() + .map(|(epoch, (stake, _))| (*epoch, *stake)) + .collect(); + let new_epoch_stakes = + EpochStakes::new(&self.stakes.read().unwrap(), leader_schedule_epoch); + info!( + "new epoch stakes, epoch: {}, stakes: {:#?}, total_stake: {}", + leader_schedule_epoch, + vote_stakes, + new_epoch_stakes.total_stake(), + ); self.epoch_stakes - .insert(leader_schedule_epoch, self.stakes.read().unwrap().clone()); + .insert(leader_schedule_epoch, new_epoch_stakes); } } @@ -2056,10 +2076,55 @@ impl Bank { self.stakes.read().unwrap().vote_accounts().clone() } + /// Get the EpochStakes for a given epoch + pub fn epoch_stakes(&self, epoch: Epoch) -> Option<&EpochStakes> { + self.epoch_stakes.get(&epoch) + } + /// vote accounts for the specific epoch along with the stake /// attributed to each account pub fn epoch_vote_accounts(&self, epoch: Epoch) -> Option<&HashMap> { - self.epoch_stakes.get(&epoch).map(Stakes::vote_accounts) + self.epoch_stakes + .get(&epoch) + .map(|epoch_stakes| Stakes::vote_accounts(epoch_stakes.stakes())) + } + + /// Get the fixed authorized voter for the given vote account for the + /// current epoch + pub fn epoch_authorized_voter(&self, vote_account: &Pubkey) -> Option<&Pubkey> { + self.epoch_stakes + .get(&self.epoch) + .expect("Epoch stakes for bank's own epoch must exist") + .epoch_authorized_voters() + .get(vote_account) + } + + /// Get the fixed set of vote accounts for the given node id for the + /// current epoch + pub fn epoch_vote_accounts_for_node_id(&self, node_id: &Pubkey) -> Option<&NodeVoteAccounts> { + self.epoch_stakes + .get(&self.epoch) + .expect("Epoch stakes for bank's own epoch must exist") + .node_id_to_vote_accounts() + .get(node_id) + } + + /// Get the fixed total stake of all vote accounts for current epoch + pub fn total_epoch_stake(&self) -> u64 { + self.epoch_stakes + .get(&self.epoch) + .expect("Epoch stakes for bank's own epoch must exist") + .total_stake() + } + + /// Get the fixed stake of the given vote account for the current epoch + pub fn epoch_vote_account_stake(&self, voting_pubkey: &Pubkey) -> u64 { + *self + .epoch_vote_accounts(self.epoch()) + .expect("Bank epoch vote accounts must contain entry for the bank's own epoch") + .get(voting_pubkey) + .map(|(stake, _)| stake) + .unwrap_or(&0) } /// given a slot, return the epoch and offset into the epoch this slot falls diff --git a/runtime/src/epoch_stakes.rs b/runtime/src/epoch_stakes.rs new file mode 100644 index 0000000000..2d0d33e2d6 --- /dev/null +++ b/runtime/src/epoch_stakes.rs @@ -0,0 +1,208 @@ +use crate::stakes::Stakes; +use serde::{Deserialize, Serialize}; +use solana_sdk::{account::Account, clock::Epoch, pubkey::Pubkey}; +use solana_vote_program::vote_state::VoteState; +use std::{collections::HashMap, sync::Arc}; + +pub type NodeIdToVoteAccounts = HashMap; +pub type EpochAuthorizedVoters = HashMap; + +#[derive(Clone, Serialize, Deserialize, Default, PartialEq)] +pub struct NodeVoteAccounts { + pub vote_accounts: Vec, + pub total_stake: u64, +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct EpochStakes { + stakes: Arc, + total_stake: u64, + node_id_to_vote_accounts: Arc, + epoch_authorized_voters: Arc, +} + +impl EpochStakes { + pub fn new(stakes: &Stakes, leader_schedule_epoch: Epoch) -> Self { + let epoch_vote_accounts = Stakes::vote_accounts(stakes); + let (total_stake, node_id_to_vote_accounts, epoch_authorized_voters) = + Self::parse_epoch_vote_accounts(&epoch_vote_accounts, leader_schedule_epoch); + Self { + stakes: Arc::new(stakes.clone()), + total_stake, + node_id_to_vote_accounts: Arc::new(node_id_to_vote_accounts), + epoch_authorized_voters: Arc::new(epoch_authorized_voters), + } + } + + pub fn stakes(&self) -> &Stakes { + &self.stakes + } + + pub fn total_stake(&self) -> u64 { + self.total_stake + } + + pub fn node_id_to_vote_accounts(&self) -> &Arc { + &self.node_id_to_vote_accounts + } + + pub fn epoch_authorized_voters(&self) -> &Arc { + &self.epoch_authorized_voters + } + + fn parse_epoch_vote_accounts( + epoch_vote_accounts: &HashMap, + leader_schedule_epoch: Epoch, + ) -> (u64, NodeIdToVoteAccounts, EpochAuthorizedVoters) { + let mut node_id_to_vote_accounts: NodeIdToVoteAccounts = HashMap::new(); + let total_stake = epoch_vote_accounts + .iter() + .map(|(_, (stake, _))| stake) + .sum(); + let epoch_authorized_voters = epoch_vote_accounts + .iter() + .filter_map(|(key, (stake, account))| { + let vote_state = VoteState::from(&account); + if vote_state.is_none() { + datapoint_warn!( + "parse_epoch_vote_accounts", + ( + "warn", + format!("Unable to get vote_state from account {}", key), + String + ), + ); + return None; + } + let vote_state = vote_state.unwrap(); + if *stake > 0 { + // Read out the authorized voters + let authorized_voter = vote_state + .authorized_voters() + .get_authorized_voter(leader_schedule_epoch) + .expect("Authorized voter for current epoch must be known"); + + let node_vote_accounts = node_id_to_vote_accounts + .entry(vote_state.node_pubkey) + .or_default(); + + node_vote_accounts.total_stake += stake; + node_vote_accounts.vote_accounts.push(*key); + + Some((*key, authorized_voter)) + } else { + None + } + }) + .collect(); + ( + total_stake, + node_id_to_vote_accounts, + epoch_authorized_voters, + ) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use solana_vote_program::vote_state::create_account_with_authorized; + use std::iter; + + struct VoteAccountInfo { + vote_account: Pubkey, + account: Account, + authorized_voter: Pubkey, + } + + #[test] + fn test_parse_epoch_vote_accounts() { + let stake_per_account = 100; + let num_vote_accounts_per_node = 2; + // Create some vote accounts for each pubkey + let vote_accounts_map: HashMap> = (0..10) + .map(|_| { + let node_id = Pubkey::new_rand(); + ( + node_id, + iter::repeat_with(|| { + let authorized_voter = Pubkey::new_rand(); + VoteAccountInfo { + vote_account: Pubkey::new_rand(), + account: create_account_with_authorized( + &node_id, + &authorized_voter, + &node_id, + 0, + 100, + ), + authorized_voter, + } + }) + .take(num_vote_accounts_per_node) + .collect(), + ) + }) + .collect(); + + let expected_authorized_voters: HashMap<_, _> = vote_accounts_map + .iter() + .flat_map(|(_, vote_accounts)| { + vote_accounts + .iter() + .map(|v| (v.vote_account, v.authorized_voter)) + }) + .collect(); + + let expected_node_id_to_vote_accounts: HashMap<_, _> = vote_accounts_map + .iter() + .map(|(node_pubkey, vote_accounts)| { + let mut vote_accounts = vote_accounts + .iter() + .map(|v| (v.vote_account)) + .collect::>(); + vote_accounts.sort(); + let node_vote_accounts = NodeVoteAccounts { + vote_accounts, + total_stake: stake_per_account * num_vote_accounts_per_node as u64, + }; + (*node_pubkey, node_vote_accounts) + }) + .collect(); + + // Create and process the vote accounts + let epoch_vote_accounts: HashMap<_, _> = vote_accounts_map + .iter() + .flat_map(|(_, vote_accounts)| { + vote_accounts + .iter() + .map(|v| (v.vote_account, (stake_per_account, v.account.clone()))) + }) + .collect(); + + let (total_stake, mut node_id_to_vote_accounts, epoch_authorized_voters) = + EpochStakes::parse_epoch_vote_accounts(&epoch_vote_accounts, 0); + + // Verify the results + node_id_to_vote_accounts + .iter_mut() + .for_each(|(_, node_vote_accounts)| node_vote_accounts.vote_accounts.sort()); + + assert!( + node_id_to_vote_accounts.len() == expected_node_id_to_vote_accounts.len() + && node_id_to_vote_accounts + .iter() + .all(|(k, v)| expected_node_id_to_vote_accounts.get(k).unwrap() == v) + ); + assert!( + epoch_authorized_voters.len() == expected_authorized_voters.len() + && epoch_authorized_voters + .iter() + .all(|(k, v)| expected_authorized_voters.get(k).unwrap() == v) + ); + assert_eq!( + total_stake, + vote_accounts_map.len() as u64 * num_vote_accounts_per_node as u64 * 100 + ); + } +} diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 45ae90ce84..bec77dc00f 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -6,6 +6,7 @@ pub mod bank; pub mod bank_client; mod blockhash_queue; pub mod bloom; +pub mod epoch_stakes; pub mod genesis_utils; pub mod loader_utils; pub mod message_processor;