diff --git a/Cargo.lock b/Cargo.lock index eba8608109..e701ac3bec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4096,12 +4096,14 @@ dependencies = [ "bs58", "bytecount", "clap", + "csv", "futures 0.3.5", "futures-util", "histogram", "itertools 0.9.0", "log 0.4.8", "regex", + "serde", "serde_json", "serde_yaml", "signal-hook", diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml index ea60c686e3..dbaeef922b 100644 --- a/ledger-tool/Cargo.toml +++ b/ledger-tool/Cargo.toml @@ -12,12 +12,14 @@ homepage = "https://solana.com/" bs58 = "0.3.1" bytecount = "0.6.0" clap = "2.33.1" +csv = "1.1.3" futures = "0.3.5" futures-util = "0.3.5" histogram = "*" itertools = "0.9.0" log = { version = "0.4.8" } regex = "1" +serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.56" serde_yaml = "0.8.13" solana-clap-utils = { path = "../clap-utils", version = "1.3.20" } diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 32cefb1aa2..34b4f0625e 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -5,6 +5,7 @@ use clap::{ use itertools::Itertools; use log::*; use regex::Regex; +use serde::Serialize; use serde_json::json; use solana_clap_utils::{ input_parsers::{cluster_type_of, pubkey_of, pubkeys_of}, @@ -20,7 +21,7 @@ use solana_ledger::{ rooted_slot_iterator::RootedSlotIterator, }; use solana_runtime::{ - bank::Bank, + bank::{Bank, RewardCalculationEvent}, bank_forks::{BankForks, CompressionType, SnapshotConfig}, hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, snapshot_utils, @@ -38,7 +39,7 @@ use solana_sdk::{ shred_version::compute_shred_version, system_program, }; -use solana_stake_program::stake_state::{self, StakeState}; +use solana_stake_program::stake_state::{self, PointValue, StakeState}; use solana_vote_program::{ self, vote_state::{self, VoteState}, @@ -1192,6 +1193,13 @@ fn main() { .help("Recalculate capitalization before warping; circumvents \ bank's out-of-sync capitalization"), ) + .arg( + Arg::with_name("csv_filename") + .long("csv-filename") + .value_name("FILENAME") + .takes_value(true) + .help("Output file in the csv format"), + ) ).subcommand( SubCommand::with_name("purge") .about("Delete a range of slots from the ledger.") @@ -2017,9 +2025,100 @@ fn main() { base_bank .lazy_rent_collection .store(true, std::sync::atomic::Ordering::Relaxed); - let warped_bank = - Bank::new_from_parent(&base_bank, base_bank.collector_id(), next_epoch); + #[derive(Default, Debug)] + struct CalculationDetail { + epochs: usize, + voter: Pubkey, + point: u128, + stake: u128, + total_stake: u64, + rent_exempt_reserve: u64, + credits: u128, + base_rewards: u64, + commission: u8, + vote_rewards: u64, + stake_rewards: u64, + activation_epoch: Epoch, + deactivation_epoch: Option, + point_value: Option, + } + use solana_stake_program::stake_state::InflationPointCalculationEvent; + let mut stake_calcuration_details: HashMap = + HashMap::new(); + let mut last_point_value = None; + let tracer = |event: &RewardCalculationEvent| { + // Currently RewardCalculationEvent enum has only Staking variant + // because only staking tracing is supported! + #[allow(irrefutable_let_patterns)] + if let RewardCalculationEvent::Staking(pubkey, event) = event { + let detail = stake_calcuration_details.entry(**pubkey).or_default(); + match event { + InflationPointCalculationEvent::CalculatedPoints( + point, + stake, + credits, + ) => { + // Don't sum for epochs where no credits are earned + if *credits > 0 { + detail.epochs += 1; + detail.point += *point; + detail.stake += *stake; + detail.credits += *credits; + } + } + InflationPointCalculationEvent::SplitRewards( + all, + voter, + staker, + point_value, + ) => { + detail.base_rewards = *all; + detail.vote_rewards = *voter; + detail.stake_rewards = *staker; + detail.point_value = Some(point_value.clone()); + // we have duplicate copies of `PointValue`s for possible + // miscalculation; do some minimum sanity check + let point_value = detail.point_value.clone(); + if point_value.is_some() { + if last_point_value.is_some() { + assert_eq!(last_point_value, point_value,); + } + last_point_value = point_value; + } + } + InflationPointCalculationEvent::Commission(commission) => { + detail.commission = *commission; + } + InflationPointCalculationEvent::RentExemptReserve(reserve) => { + detail.rent_exempt_reserve = *reserve; + } + InflationPointCalculationEvent::Delegation(delegation) => { + detail.voter = delegation.voter_pubkey; + detail.total_stake = delegation.stake; + detail.activation_epoch = delegation.activation_epoch; + if delegation.deactivation_epoch < Epoch::max_value() { + detail.deactivation_epoch = + Some(delegation.deactivation_epoch); + } + } + } + } + }; + let warped_bank = Bank::new_from_parent_with_tracer( + &base_bank, + base_bank.collector_id(), + next_epoch, + tracer, + ); warped_bank.freeze(); + let mut csv_writer = if arg_matches.is_present("csv_filename") { + let csv_filename = + value_t_or_exit!(arg_matches, "csv_filename", String); + let file = File::create(&csv_filename).unwrap(); + Some(csv::WriterBuilder::new().from_writer(file)) + } else { + None + }; println!("Slot: {} => {}", base_bank.slot(), warped_bank.slot()); println!("Epoch: {} => {}", base_bank.epoch(), warped_bank.epoch()); @@ -2041,9 +2140,10 @@ fn main() { ); let mut overall_delta = 0; + let modified_accounts = warped_bank.get_all_accounts_modified_since_parent(); - let mut sorted_accounts = modified_accounts + let mut rewarded_accounts = modified_accounts .iter() .map(|(pubkey, account)| { ( @@ -2056,32 +2156,133 @@ fn main() { ) }) .collect::>(); - sorted_accounts.sort_unstable_by_key(|(pubkey, account, base_lamports)| { - ( - account.owner, - *base_lamports, - account.lamports - base_lamports, - *pubkey, + rewarded_accounts.sort_unstable_by_key( + |(pubkey, account, base_lamports)| { + ( + account.owner, + *base_lamports, + account.lamports - base_lamports, + *pubkey, + ) + }, + ); + + let mut unchanged_accounts = stake_calcuration_details + .keys() + .collect::>() + .difference( + &rewarded_accounts + .iter() + .map(|(pubkey, ..)| *pubkey) + .collect(), ) + .map(|pubkey| (**pubkey, warped_bank.get_account(pubkey).unwrap())) + .collect::>(); + unchanged_accounts.sort_unstable_by_key(|(pubkey, account)| { + (account.owner, account.lamports, *pubkey) }); - for (pubkey, warped_account, _) in sorted_accounts { + let unchanged_accounts = unchanged_accounts.into_iter(); + + let rewarded_accounts = rewarded_accounts + .into_iter() + .map(|(pubkey, account, ..)| (*pubkey, account.clone())); + + let all_accounts = unchanged_accounts.chain(rewarded_accounts); + for (pubkey, warped_account) in all_accounts { + // Don't output sysvars; it's always updated but not related to + // inflation. + if solana_sdk::sysvar::is_sysvar_id(&pubkey) { + continue; + } + if let Some(base_account) = base_bank.get_account(&pubkey) { - if base_account.lamports != warped_account.lamports { - let delta = warped_account.lamports - base_account.lamports; - println!( - "{:<45}({}): {} => {} (+{} {:>4.9}%)", - format!("{}", pubkey), // format! is needed to pad/justify correctly. - base_account.owner, - Sol(base_account.lamports), - Sol(warped_account.lamports), - Sol(delta), - ((warped_account.lamports as f64) - / (base_account.lamports as f64) - * 100_f64) - - 100_f64, - ); - overall_delta += delta; + let delta = warped_account.lamports - base_account.lamports; + let detail = stake_calcuration_details.get(&pubkey); + println!( + "{:<45}({}): {} => {} (+{} {:>4.9}%) {:?}", + format!("{}", pubkey), // format! is needed to pad/justify correctly. + base_account.owner, + Sol(base_account.lamports), + Sol(warped_account.lamports), + Sol(delta), + ((warped_account.lamports as f64) + / (base_account.lamports as f64) + * 100_f64) + - 100_f64, + detail, + ); + if let Some(ref mut csv_writer) = csv_writer { + #[derive(Serialize)] + struct InflationRecord { + account: String, + owner: String, + old_balance: u64, + new_balance: u64, + data_size: usize, + delegation: String, + effective_stake: String, + delegated_stake: String, + rent_exempt_reserve: String, + activation_epoch: String, + deactivation_epoch: String, + earned_epochs: String, + earned_credits: String, + base_rewards: String, + stake_rewards: String, + vote_rewards: String, + commission: String, + cluster_rewards: String, + cluster_points: String, + }; + fn format_or_na( + data: Option, + ) -> String { + data.map(|data| format!("{}", data)) + .unwrap_or_else(|| "N/A".to_owned()) + }; + let record = InflationRecord { + account: format!("{}", pubkey), + owner: format!("{}", base_account.owner), + old_balance: base_account.lamports, + new_balance: warped_account.lamports, + data_size: base_account.data.len(), + delegation: format_or_na(detail.map(|d| d.voter)), + effective_stake: format_or_na(detail.map(|d| d.stake)), + delegated_stake: format_or_na( + detail.map(|d| d.total_stake), + ), + rent_exempt_reserve: format_or_na( + detail.map(|d| d.rent_exempt_reserve), + ), + activation_epoch: format_or_na(detail.map(|d| { + if d.activation_epoch < Epoch::max_value() { + d.activation_epoch + } else { + // bootstraped + 0 + } + })), + deactivation_epoch: format_or_na( + detail.and_then(|d| d.deactivation_epoch), + ), + earned_epochs: format_or_na(detail.map(|d| d.epochs)), + earned_credits: format_or_na(detail.map(|d| d.credits)), + base_rewards: format_or_na(detail.map(|d| d.base_rewards)), + stake_rewards: format_or_na( + detail.map(|d| d.stake_rewards), + ), + vote_rewards: format_or_na(detail.map(|d| d.vote_rewards)), + commission: format_or_na(detail.map(|d| d.commission)), + cluster_rewards: format_or_na( + last_point_value.as_ref().map(|pv| pv.rewards), + ), + cluster_points: format_or_na( + last_point_value.as_ref().map(|pv| pv.points), + ), + }; + csv_writer.serialize(&record).unwrap(); } + overall_delta += delta; } else { error!("new account!?: {}", pubkey); } diff --git a/programs/stake/src/stake_state.rs b/programs/stake/src/stake_state.rs index d87e316c21..00bb5e9d6b 100644 --- a/programs/stake/src/stake_state.rs +++ b/programs/stake/src/stake_state.rs @@ -30,6 +30,19 @@ pub enum StakeState { RewardsPool, } +#[derive(Debug)] +pub enum InflationPointCalculationEvent { + CalculatedPoints(u128, u128, u128), + SplitRewards(u64, u64, u64, PointValue), + RentExemptReserve(u64), + Delegation(Delegation), + Commission(u8), +} + +fn null_tracer() -> Option { + None:: +} + impl Default for StakeState { fn default() -> Self { StakeState::Uninitialized @@ -368,6 +381,7 @@ impl Authorized { /// and the total points over which those lamports /// are to be distributed // basically read as rewards/points, but in integers instead of as an f64 +#[derive(Clone, Debug, PartialEq)] pub struct PointValue { pub rewards: u64, // lamports to split pub points: u128, // over these points @@ -383,21 +397,28 @@ impl Stake { point_value: &PointValue, vote_state: &VoteState, stake_history: Option<&StakeHistory>, + inflation_point_calc_tracer: &mut Option, ) -> Option<(u64, u64)> { - self.calculate_rewards(point_value, vote_state, stake_history) - .map(|(stakers_reward, voters_reward, credits_observed)| { - self.credits_observed = credits_observed; - self.delegation.stake += stakers_reward; - (stakers_reward, voters_reward) - }) + self.calculate_rewards( + point_value, + vote_state, + stake_history, + inflation_point_calc_tracer, + ) + .map(|(stakers_reward, voters_reward, credits_observed)| { + self.credits_observed = credits_observed; + self.delegation.stake += stakers_reward; + (stakers_reward, voters_reward) + }) } pub fn calculate_points( &self, vote_state: &VoteState, stake_history: Option<&StakeHistory>, + inflation_point_calc_tracer: &mut Option, ) -> u128 { - self.calculate_points_and_credits(vote_state, stake_history) + self.calculate_points_and_credits(vote_state, stake_history, inflation_point_calc_tracer) .0 } @@ -408,6 +429,7 @@ impl Stake { &self, new_vote_state: &VoteState, stake_history: Option<&StakeHistory>, + inflation_point_calc_tracer: &mut Option, ) -> (u128, u64) { // if there is no newer credits since observed, return no point if new_vote_state.credits() <= self.credits_observed { @@ -442,6 +464,14 @@ impl Stake { // finally calculate points for this epoch points += stake * earned_credits; + + if let Some(inflation_point_calc_tracer) = inflation_point_calc_tracer { + inflation_point_calc_tracer(&InflationPointCalculationEvent::CalculatedPoints( + points, + stake, + earned_credits, + )); + } } (points, new_credits_observed) @@ -458,9 +488,13 @@ impl Stake { point_value: &PointValue, vote_state: &VoteState, stake_history: Option<&StakeHistory>, + inflation_point_calc_tracer: &mut Option, ) -> Option<(u64, u64, u64)> { - let (points, credits_observed) = - self.calculate_points_and_credits(vote_state, stake_history); + let (points, credits_observed) = self.calculate_points_and_credits( + vote_state, + stake_history, + inflation_point_calc_tracer, + ); if points == 0 || point_value.points == 0 { return None; @@ -479,6 +513,14 @@ impl Stake { return None; } let (voter_rewards, staker_rewards, is_split) = vote_state.commission_split(rewards); + if let Some(inflation_point_calc_tracer) = inflation_point_calc_tracer { + inflation_point_calc_tracer(&InflationPointCalculationEvent::SplitRewards( + rewards, + voter_rewards, + staker_rewards, + (*point_value).clone(), + )); + } if (voter_rewards == 0 || staker_rewards == 0) && is_split { // don't collect if we lose a whole lamport somewhere @@ -971,14 +1013,26 @@ pub fn redeem_rewards( vote_account: &mut Account, point_value: &PointValue, stake_history: Option<&StakeHistory>, + inflation_point_calc_tracer: &mut Option, ) -> Result<(u64, u64), InstructionError> { if let StakeState::Stake(meta, mut stake) = stake_account.state()? { let vote_state: VoteState = StateMut::::state(vote_account)?.convert_to_current(); + if let Some(inflation_point_calc_tracer) = inflation_point_calc_tracer { + inflation_point_calc_tracer(&InflationPointCalculationEvent::RentExemptReserve( + meta.rent_exempt_reserve, + )); + inflation_point_calc_tracer(&InflationPointCalculationEvent::Commission( + vote_state.commission, + )); + } - if let Some((stakers_reward, voters_reward)) = - stake.redeem_rewards(point_value, &vote_state, stake_history) - { + if let Some((stakers_reward, voters_reward)) = stake.redeem_rewards( + point_value, + &vote_state, + stake_history, + inflation_point_calc_tracer, + ) { stake_account.lamports += stakers_reward; vote_account.lamports += voters_reward; @@ -1003,7 +1057,7 @@ pub fn calculate_points( let vote_state: VoteState = StateMut::::state(vote_account)?.convert_to_current(); - Ok(stake.calculate_points(&vote_state, stake_history)) + Ok(stake.calculate_points(&vote_state, stake_history, &mut null_tracer())) } else { Err(InstructionError::InvalidAccountData) } @@ -2439,7 +2493,8 @@ mod tests { points: 1 }, &vote_state, - None + None, + &mut null_tracer(), ) ); @@ -2456,7 +2511,8 @@ mod tests { points: 1 }, &vote_state, - None + None, + &mut null_tracer(), ) ); @@ -2490,7 +2546,8 @@ mod tests { points: 1 }, &vote_state, - None + None, + &mut null_tracer(), ) ); @@ -2504,7 +2561,7 @@ mod tests { // no overflow on points assert_eq!( u128::from(stake.delegation.stake) * epoch_slots, - stake.calculate_points(&vote_state, None) + stake.calculate_points(&vote_state, None, &mut null_tracer()) ); } @@ -2530,7 +2587,8 @@ mod tests { points: 1 }, &vote_state, - None + None, + &mut null_tracer(), ) ); @@ -2547,7 +2605,8 @@ mod tests { points: 2 // all his }, &vote_state, - None + None, + &mut null_tracer(), ) ); @@ -2561,7 +2620,8 @@ mod tests { points: 1 }, &vote_state, - None + None, + &mut null_tracer(), ) ); @@ -2578,7 +2638,8 @@ mod tests { points: 2 }, &vote_state, - None + None, + &mut null_tracer(), ) ); @@ -2593,7 +2654,8 @@ mod tests { points: 2 }, &vote_state, - None + None, + &mut null_tracer(), ) ); @@ -2614,7 +2676,8 @@ mod tests { points: 4 }, &vote_state, - None + None, + &mut null_tracer(), ) ); @@ -2629,7 +2692,8 @@ mod tests { points: 4 }, &vote_state, - None + None, + &mut null_tracer(), ) ); vote_state.commission = 99; @@ -2641,7 +2705,8 @@ mod tests { points: 4 }, &vote_state, - None + None, + &mut null_tracer(), ) ); } diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 0f36a9a75b..a2792737c6 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -66,7 +66,9 @@ use solana_sdk::{ timing::years_as_slots, transaction::{self, Result, Transaction, TransactionError}, }; -use solana_stake_program::stake_state::{self, Delegation, PointValue}; +use solana_stake_program::stake_state::{ + self, Delegation, InflationPointCalculationEvent, PointValue, +}; use solana_vote_program::{vote_instruction::VoteInstruction, vote_state::VoteState}; use std::{ cell::RefCell, @@ -575,6 +577,15 @@ impl PartialEq for Bank { } } +#[derive(Debug)] +pub enum RewardCalculationEvent<'a, 'b> { + Staking(&'a Pubkey, &'b InflationPointCalculationEvent), +} + +fn null_tracer() -> Option { + None:: +} + #[derive(Debug, PartialEq, Serialize, Deserialize, AbiExample, Clone, Copy)] pub struct RewardInfo { pub reward_type: RewardType, @@ -767,6 +778,24 @@ impl Bank { /// Create a new bank that points to an immutable checkpoint of another bank. pub fn new_from_parent(parent: &Arc, collector_id: &Pubkey, slot: Slot) -> Self { + Self::_new_from_parent(parent, collector_id, slot, &mut null_tracer()) + } + + pub fn new_from_parent_with_tracer( + parent: &Arc, + collector_id: &Pubkey, + slot: Slot, + reward_calc_tracer: impl FnMut(&RewardCalculationEvent), + ) -> Self { + Self::_new_from_parent(parent, collector_id, slot, &mut Some(reward_calc_tracer)) + } + + fn _new_from_parent( + parent: &Arc, + collector_id: &Pubkey, + slot: Slot, + reward_calc_tracer: &mut Option, + ) -> Self { parent.freeze(); assert_ne!(slot, parent.slot()); @@ -858,7 +887,7 @@ impl Bank { } new.update_slot_hashes(); - new.update_rewards(parent.epoch()); + new.update_rewards(parent.epoch(), reward_calc_tracer); new.update_stake_history(Some(parent.epoch())); new.update_clock(); new.update_fees(); @@ -1200,7 +1229,11 @@ impl Bank { } // update rewards based on the previous epoch - fn update_rewards(&mut self, prev_epoch: Epoch) { + fn update_rewards( + &mut self, + prev_epoch: Epoch, + reward_calc_tracer: &mut Option, + ) { if prev_epoch == self.epoch() { return; } @@ -1226,7 +1259,8 @@ impl Bank { let old_vote_balance_and_staked = self.stakes.read().unwrap().vote_balance_and_staked(); - let validator_point_value = self.pay_validator_rewards(validator_rewards); + let validator_point_value = + self.pay_validator_rewards(validator_rewards, reward_calc_tracer); if !self .feature_set @@ -1298,7 +1332,10 @@ impl Bank { /// returns a map (has to be copied) of loaded /// ( Vec<(staker info)> (voter account) ) keyed by voter pubkey /// - fn stake_delegation_accounts(&self) -> HashMap, Account)> { + fn stake_delegation_accounts( + &self, + reward_calc_tracer: &mut Option, + ) -> HashMap, Account)> { let mut accounts = HashMap::new(); self.stakes @@ -1315,6 +1352,12 @@ impl Bank { let entry = accounts .entry(delegation.voter_pubkey) .or_insert((Vec::new(), vote_account)); + if let Some(reward_calc_tracer) = reward_calc_tracer { + reward_calc_tracer(&RewardCalculationEvent::Staking( + stake_pubkey, + &InflationPointCalculationEvent::Delegation(*delegation), + )); + } entry.0.push((*stake_pubkey, stake_account)); } (_, _) => {} @@ -1326,10 +1369,14 @@ impl Bank { /// iterate over all stakes, redeem vote credits for each stake we can /// successfully load and parse, return the lamport value of one point - fn pay_validator_rewards(&mut self, rewards: u64) -> f64 { + fn pay_validator_rewards( + &mut self, + rewards: u64, + reward_calc_tracer: &mut Option, + ) -> f64 { let stake_history = self.stakes.read().unwrap().history().clone(); - let mut stake_delegation_accounts = self.stake_delegation_accounts(); + let mut stake_delegation_accounts = self.stake_delegation_accounts(reward_calc_tracer); let points: u128 = stake_delegation_accounts .iter() @@ -1357,11 +1404,20 @@ impl Bank { let voters_account_pre_balance = vote_account.lamports; for (stake_pubkey, stake_account) in stake_group.iter_mut() { + // curry closure to add the contextual stake_pubkey + let mut reward_calc_tracer = reward_calc_tracer.as_mut().map(|outer| { + let stake_pubkey = *stake_pubkey; + // inner + move |inner_event: &_| { + outer(&RewardCalculationEvent::Staking(&stake_pubkey, inner_event)) + } + }); let redeemed = stake_state::redeem_rewards( stake_account, vote_account, &point_value, Some(&stake_history), + &mut reward_calc_tracer.as_mut(), ); if let Ok((stakers_reward, _voters_reward)) = redeemed { self.store_account(&stake_pubkey, &stake_account); @@ -5842,7 +5898,7 @@ mod tests { bank.add_account_and_update_capitalization(&vote_id, &vote_account); let validator_points: u128 = bank - .stake_delegation_accounts() + .stake_delegation_accounts(&mut null_tracer()) .iter() .flat_map(|(_vote_pubkey, (stake_group, vote_account))| { stake_group