Convert Banks (#9033)

* Store and compute needed state in EpochStakes struct
Co-authored-by: Carl <carl@solana.com>

ledger/src/snapshot_utils.rs

@@ -8,7 +8,7 @@ use solana_measure::measure::Measure;
 use solana_runtime::{
     accounts_db::{SnapshotStorage, SnapshotStorages},
     bank::{
-        self, deserialize_from_snapshot, Bank, BankRcSerialize, BankSlotDelta,
+        self, bank_1_0::Bank1_0, deserialize_from_snapshot, Bank, BankRcSerialize, BankSlotDelta,
         MAX_SNAPSHOT_DATA_FILE_SIZE,
     },
 };
@@ -30,7 +30,8 @@ pub const TAR_SNAPSHOTS_DIR: &str = "snapshots";
 pub const TAR_ACCOUNTS_DIR: &str = "accounts";
 pub const TAR_VERSION_FILE: &str = "version";
 
-pub const SNAPSHOT_VERSION: &str = "1.0.0";
+pub const SNAPSHOT_VERSION_1_0: &str = "1.0.0";
+pub const SNAPSHOT_VERSION: &str = "1.1.0";
 
 #[derive(PartialEq, Ord, Eq, Debug)]
 pub struct SlotSnapshotPaths {
@@ -593,6 +594,10 @@ where
         MAX_SNAPSHOT_DATA_FILE_SIZE,
         |stream| {
             let mut bank: Bank = match snapshot_version {
+                SNAPSHOT_VERSION_1_0 => {
+                    let bank_1_0: Bank1_0 = deserialize_from_snapshot(stream.by_ref())?;
+                    bank_1_0.convert_to_current()
+                }
                 SNAPSHOT_VERSION => deserialize_from_snapshot(stream.by_ref())?,
                 _ => {
                     return Err(get_io_error(&format!(
@@ -602,6 +607,7 @@ where
                 }
             };
             info!("Rebuilding accounts...");
+
             let rc = bank::BankRc::from_stream(
                 account_paths,
                 bank.slot(),

runtime/src/bank/bank_1_0.rs

@@ -0,0 +1,220 @@
+//! The `bank` module tracks client accounts and the progress of on-chain
+//! programs. It offers a high-level API that signs transactions
+//! on behalf of the caller, and a low-level API for when they have
+//! already been signed and verified.
+use crate::{
+    bank::{Bank, BankRc, EnteredEpochCallback, StatusCacheRc},
+    blockhash_queue::BlockhashQueue,
+    epoch_stakes::EpochStakes,
+    message_processor::MessageProcessor,
+    rent_collector::RentCollector,
+    serde_utils::{
+        deserialize_atomicbool, deserialize_atomicu64, serialize_atomicbool, serialize_atomicu64,
+    },
+    stakes::Stakes,
+    storage_utils::StorageAccounts,
+};
+use serde::{Deserialize, Serialize};
+use solana_sdk::{
+    clock::{Epoch, Slot, UnixTimestamp},
+    epoch_schedule::EpochSchedule,
+    fee_calculator::{FeeCalculator, FeeRateGovernor},
+    hard_forks::HardForks,
+    hash::Hash,
+    inflation::Inflation,
+    pubkey::Pubkey,
+};
+use std::{
+    collections::HashMap,
+    sync::atomic::{AtomicBool, AtomicU64},
+    sync::{Arc, RwLock},
+};
+
+/// Manager for the state of all accounts and programs after processing its entries.
+#[derive(Deserialize, Serialize)]
+pub struct Bank1_0 {
+    /// References to accounts, parent and signature status
+    #[serde(skip)]
+    pub rc: BankRc,
+
+    #[serde(skip)]
+    pub src: StatusCacheRc,
+
+    /// FIFO queue of `recent_blockhash` items
+    pub blockhash_queue: RwLock<BlockhashQueue>,
+
+    /// The set of parents including this bank
+    pub ancestors: HashMap<Slot, usize>,
+
+    /// Hash of this Bank's state. Only meaningful after freezing.
+    pub hash: RwLock<Hash>,
+
+    /// Hash of this Bank's parent's state
+    pub parent_hash: Hash,
+
+    /// parent's slot
+    pub parent_slot: Slot,
+
+    /// slots to hard fork at
+    pub hard_forks: Arc<RwLock<HardForks>>,
+
+    /// The number of transactions processed without error
+    #[serde(serialize_with = "serialize_atomicu64")]
+    #[serde(deserialize_with = "deserialize_atomicu64")]
+    pub transaction_count: AtomicU64,
+
+    /// Bank tick height
+    #[serde(serialize_with = "serialize_atomicu64")]
+    #[serde(deserialize_with = "deserialize_atomicu64")]
+    pub tick_height: AtomicU64,
+
+    /// The number of signatures from valid transactions in this slot
+    #[serde(serialize_with = "serialize_atomicu64")]
+    #[serde(deserialize_with = "deserialize_atomicu64")]
+    pub signature_count: AtomicU64,
+
+    /// Total capitalization, used to calculate inflation
+    #[serde(serialize_with = "serialize_atomicu64")]
+    #[serde(deserialize_with = "deserialize_atomicu64")]
+    pub capitalization: AtomicU64,
+
+    // Bank max_tick_height
+    pub max_tick_height: u64,
+
+    /// The number of hashes in each tick. None value means hashing is disabled.
+    pub hashes_per_tick: Option<u64>,
+
+    /// The number of ticks in each slot.
+    pub ticks_per_slot: u64,
+
+    /// length of a slot in ns
+    pub ns_per_slot: u128,
+
+    /// genesis time, used for computed clock
+    pub genesis_creation_time: UnixTimestamp,
+
+    /// The number of slots per year, used for inflation
+    pub slots_per_year: f64,
+
+    /// The number of slots per Storage segment
+    pub slots_per_segment: u64,
+
+    /// Bank slot (i.e. block)
+    pub slot: Slot,
+
+    /// Bank epoch
+    pub epoch: Epoch,
+
+    /// Bank block_height
+    pub block_height: u64,
+
+    /// The pubkey to send transactions fees to.
+    pub collector_id: Pubkey,
+
+    /// Fees that have been collected
+    #[serde(serialize_with = "serialize_atomicu64")]
+    #[serde(deserialize_with = "deserialize_atomicu64")]
+    pub collector_fees: AtomicU64,
+
+    /// Latest transaction fees for transactions processed by this bank
+    pub fee_calculator: FeeCalculator,
+
+    /// Track cluster signature throughput and adjust fee rate
+    pub fee_rate_governor: FeeRateGovernor,
+
+    /// Rent that have been collected
+    #[serde(serialize_with = "serialize_atomicu64")]
+    #[serde(deserialize_with = "deserialize_atomicu64")]
+    pub collected_rent: AtomicU64,
+
+    /// latest rent collector, knows the epoch
+    pub rent_collector: RentCollector,
+
+    /// initialized from genesis
+    pub epoch_schedule: EpochSchedule,
+
+    /// inflation specs
+    pub inflation: Arc<RwLock<Inflation>>,
+
+    /// cache of vote_account and stake_account state for this fork
+    pub stakes: RwLock<Stakes>,
+
+    /// cache of validator and archiver storage accounts for this fork
+    pub storage_accounts: RwLock<StorageAccounts>,
+
+    /// staked nodes on epoch boundaries, saved off when a bank.slot() is at
+    ///   a leader schedule calculation boundary
+    pub epoch_stakes: HashMap<Epoch, Stakes>,
+
+    /// A boolean reflecting whether any entries were recorded into the PoH
+    /// stream for the slot == self.slot
+    #[serde(serialize_with = "serialize_atomicbool")]
+    #[serde(deserialize_with = "deserialize_atomicbool")]
+    pub is_delta: AtomicBool,
+
+    /// The Message processor
+    pub message_processor: MessageProcessor,
+
+    /// Callback to be notified when a bank enters a new Epoch
+    /// (used to adjust cluster features over time)
+    #[serde(skip)]
+    pub entered_epoch_callback: Arc<RwLock<Option<EnteredEpochCallback>>>,
+
+    /// Last time when the cluster info vote listener has synced with this bank
+    #[serde(skip)]
+    pub last_vote_sync: AtomicU64,
+
+    /// Rewards that were paid out immediately after this bank was created
+    #[serde(skip)]
+    pub rewards: Option<Vec<(Pubkey, i64)>>,
+}
+
+impl Bank1_0 {
+    pub fn convert_to_current(self) -> Bank {
+        let old_epoch_stakes = self.epoch_stakes;
+        let epoch_stakes = old_epoch_stakes
+            .iter()
+            .map(|(epoch, stakes)| (*epoch, EpochStakes::new(&stakes, *epoch)))
+            .collect();
+        Bank {
+            rc: self.rc,
+            src: self.src,
+            blockhash_queue: self.blockhash_queue,
+            ancestors: self.ancestors,
+            hash: self.hash,
+            parent_hash: self.parent_hash,
+            parent_slot: self.parent_slot,
+            hard_forks: self.hard_forks,
+            transaction_count: self.transaction_count,
+            tick_height: self.tick_height,
+            signature_count: self.signature_count,
+            capitalization: self.capitalization,
+            max_tick_height: self.max_tick_height,
+            hashes_per_tick: self.hashes_per_tick,
+            ticks_per_slot: self.ticks_per_slot,
+            ns_per_slot: self.ns_per_slot,
+            genesis_creation_time: self.genesis_creation_time,
+            slots_per_year: self.slots_per_year,
+            slots_per_segment: self.slots_per_segment,
+            slot: self.slot,
+            epoch: self.epoch,
+            block_height: self.block_height,
+            collector_id: self.collector_id,
+            collector_fees: self.collector_fees,
+            fee_calculator: self.fee_calculator,
+            fee_rate_governor: self.fee_rate_governor,
+            collected_rent: self.collected_rent,
+            rent_collector: self.rent_collector,
+            epoch_schedule: self.epoch_schedule,
+            inflation: self.inflation,
+            stakes: self.stakes,
+            storage_accounts: self.storage_accounts,
+            epoch_stakes,
+            is_delta: self.is_delta,
+            message_processor: self.message_processor,
+            entered_epoch_callback: self.entered_epoch_callback,
+            last_vote_sync: self.last_vote_sync,
+            rewards: self.rewards,
+        }
+    }
+}

runtime/src/bank/mod.rs

@@ -6,6 +6,7 @@ use crate::{
     accounts::{Accounts, TransactionAccounts, TransactionLoadResult, TransactionLoaders},
     accounts_db::{AccountsDBSerialize, ErrorCounters, SnapshotStorage, SnapshotStorages},
     blockhash_queue::BlockhashQueue,
+    epoch_stakes::{EpochStakes, NodeVoteAccounts},
     message_processor::{MessageProcessor, ProcessInstruction},
     nonce_utils,
     rent_collector::RentCollector,
@@ -60,6 +61,8 @@ use std::{
     sync::{Arc, RwLock, RwLockReadGuard},
 };
 
+pub mod bank_1_0;
+
 pub const SECONDS_PER_YEAR: f64 = 365.25 * 24.0 * 60.0 * 60.0;
 pub const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB
 
@@ -323,7 +326,7 @@ pub struct Bank {
 
     /// staked nodes on epoch boundaries, saved off when a bank.slot() is at
     ///   a leader schedule calculation boundary
-    epoch_stakes: HashMap<Epoch, Stakes>,
+    epoch_stakes: HashMap<Epoch, EpochStakes>,
 
     /// A boolean reflecting whether any entries were recorded into the PoH
     /// stream for the slot == self.slot
@@ -380,7 +383,8 @@ impl Bank {
         {
             let stakes = bank.stakes.read().unwrap();
             for epoch in 0..=bank.get_leader_schedule_epoch(bank.slot) {
-                bank.epoch_stakes.insert(epoch, stakes.clone());
+                bank.epoch_stakes
+                    .insert(epoch, EpochStakes::new(&stakes, epoch));
             }
             bank.update_stake_history(None);
         }
@@ -592,8 +596,24 @@ impl Bank {
                 epoch >= leader_schedule_epoch.saturating_sub(MAX_LEADER_SCHEDULE_STAKES)
             });
 
+            let vote_stakes: HashMap<_, _> = self
+                .stakes
+                .read()
+                .unwrap()
+                .vote_accounts()
+                .iter()
+                .map(|(epoch, (stake, _))| (*epoch, *stake))
+                .collect();
+            let new_epoch_stakes =
+                EpochStakes::new(&self.stakes.read().unwrap(), leader_schedule_epoch);
+            info!(
+                "new epoch stakes, epoch: {}, stakes: {:#?}, total_stake: {}",
+                leader_schedule_epoch,
+                vote_stakes,
+                new_epoch_stakes.total_stake(),
+            );
             self.epoch_stakes
-                .insert(leader_schedule_epoch, self.stakes.read().unwrap().clone());
+                .insert(leader_schedule_epoch, new_epoch_stakes);
         }
     }
 
@@ -2056,10 +2076,55 @@ impl Bank {
         self.stakes.read().unwrap().vote_accounts().clone()
     }
 
+    /// Get the EpochStakes for a given epoch
+    pub fn epoch_stakes(&self, epoch: Epoch) -> Option<&EpochStakes> {
+        self.epoch_stakes.get(&epoch)
+    }
+
     /// vote accounts for the specific epoch along with the stake
     ///   attributed to each account
     pub fn epoch_vote_accounts(&self, epoch: Epoch) -> Option<&HashMap<Pubkey, (u64, Account)>> {
-        self.epoch_stakes.get(&epoch).map(Stakes::vote_accounts)
+        self.epoch_stakes
+            .get(&epoch)
+            .map(|epoch_stakes| Stakes::vote_accounts(epoch_stakes.stakes()))
+    }
+
+    /// Get the fixed authorized voter for the given vote account for the
+    /// current epoch
+    pub fn epoch_authorized_voter(&self, vote_account: &Pubkey) -> Option<&Pubkey> {
+        self.epoch_stakes
+            .get(&self.epoch)
+            .expect("Epoch stakes for bank's own epoch must exist")
+            .epoch_authorized_voters()
+            .get(vote_account)
+    }
+
+    /// Get the fixed set of vote accounts for the given node id for the
+    /// current epoch
+    pub fn epoch_vote_accounts_for_node_id(&self, node_id: &Pubkey) -> Option<&NodeVoteAccounts> {
+        self.epoch_stakes
+            .get(&self.epoch)
+            .expect("Epoch stakes for bank's own epoch must exist")
+            .node_id_to_vote_accounts()
+            .get(node_id)
+    }
+
+    /// Get the fixed total stake of all vote accounts for current epoch
+    pub fn total_epoch_stake(&self) -> u64 {
+        self.epoch_stakes
+            .get(&self.epoch)
+            .expect("Epoch stakes for bank's own epoch must exist")
+            .total_stake()
+    }
+
+    /// Get the fixed stake of the given vote account for the current epoch
+    pub fn epoch_vote_account_stake(&self, voting_pubkey: &Pubkey) -> u64 {
+        *self
+            .epoch_vote_accounts(self.epoch())
+            .expect("Bank epoch vote accounts must contain entry for the bank's own epoch")
+            .get(voting_pubkey)
+            .map(|(stake, _)| stake)
+            .unwrap_or(&0)
     }
 
     /// given a slot, return the epoch and offset into the epoch this slot falls

runtime/src/epoch_stakes.rs

@@ -0,0 +1,208 @@
+use crate::stakes::Stakes;
+use serde::{Deserialize, Serialize};
+use solana_sdk::{account::Account, clock::Epoch, pubkey::Pubkey};
+use solana_vote_program::vote_state::VoteState;
+use std::{collections::HashMap, sync::Arc};
+
+pub type NodeIdToVoteAccounts = HashMap<Pubkey, NodeVoteAccounts>;
+pub type EpochAuthorizedVoters = HashMap<Pubkey, Pubkey>;
+
+#[derive(Clone, Serialize, Deserialize, Default, PartialEq)]
+pub struct NodeVoteAccounts {
+    pub vote_accounts: Vec<Pubkey>,
+    pub total_stake: u64,
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct EpochStakes {
+    stakes: Arc<Stakes>,
+    total_stake: u64,
+    node_id_to_vote_accounts: Arc<NodeIdToVoteAccounts>,
+    epoch_authorized_voters: Arc<EpochAuthorizedVoters>,
+}
+
+impl EpochStakes {
+    pub fn new(stakes: &Stakes, leader_schedule_epoch: Epoch) -> Self {
+        let epoch_vote_accounts = Stakes::vote_accounts(stakes);
+        let (total_stake, node_id_to_vote_accounts, epoch_authorized_voters) =
+            Self::parse_epoch_vote_accounts(&epoch_vote_accounts, leader_schedule_epoch);
+        Self {
+            stakes: Arc::new(stakes.clone()),
+            total_stake,
+            node_id_to_vote_accounts: Arc::new(node_id_to_vote_accounts),
+            epoch_authorized_voters: Arc::new(epoch_authorized_voters),
+        }
+    }
+
+    pub fn stakes(&self) -> &Stakes {
+        &self.stakes
+    }
+
+    pub fn total_stake(&self) -> u64 {
+        self.total_stake
+    }
+
+    pub fn node_id_to_vote_accounts(&self) -> &Arc<NodeIdToVoteAccounts> {
+        &self.node_id_to_vote_accounts
+    }
+
+    pub fn epoch_authorized_voters(&self) -> &Arc<EpochAuthorizedVoters> {
+        &self.epoch_authorized_voters
+    }
+
+    fn parse_epoch_vote_accounts(
+        epoch_vote_accounts: &HashMap<Pubkey, (u64, Account)>,
+        leader_schedule_epoch: Epoch,
+    ) -> (u64, NodeIdToVoteAccounts, EpochAuthorizedVoters) {
+        let mut node_id_to_vote_accounts: NodeIdToVoteAccounts = HashMap::new();
+        let total_stake = epoch_vote_accounts
+            .iter()
+            .map(|(_, (stake, _))| stake)
+            .sum();
+        let epoch_authorized_voters = epoch_vote_accounts
+            .iter()
+            .filter_map(|(key, (stake, account))| {
+                let vote_state = VoteState::from(&account);
+                if vote_state.is_none() {
+                    datapoint_warn!(
+                        "parse_epoch_vote_accounts",
+                        (
+                            "warn",
+                            format!("Unable to get vote_state from account {}", key),
+                            String
+                        ),
+                    );
+                    return None;
+                }
+                let vote_state = vote_state.unwrap();
+                if *stake > 0 {
+                    // Read out the authorized voters
+                    let authorized_voter = vote_state
+                        .authorized_voters()
+                        .get_authorized_voter(leader_schedule_epoch)
+                        .expect("Authorized voter for current epoch must be known");
+
+                    let node_vote_accounts = node_id_to_vote_accounts
+                        .entry(vote_state.node_pubkey)
+                        .or_default();
+
+                    node_vote_accounts.total_stake += stake;
+                    node_vote_accounts.vote_accounts.push(*key);
+
+                    Some((*key, authorized_voter))
+                } else {
+                    None
+                }
+            })
+            .collect();
+        (
+            total_stake,
+            node_id_to_vote_accounts,
+            epoch_authorized_voters,
+        )
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::*;
+    use solana_vote_program::vote_state::create_account_with_authorized;
+    use std::iter;
+
+    struct VoteAccountInfo {
+        vote_account: Pubkey,
+        account: Account,
+        authorized_voter: Pubkey,
+    }
+
+    #[test]
+    fn test_parse_epoch_vote_accounts() {
+        let stake_per_account = 100;
+        let num_vote_accounts_per_node = 2;
+        // Create some vote accounts for each pubkey
+        let vote_accounts_map: HashMap<Pubkey, Vec<VoteAccountInfo>> = (0..10)
+            .map(|_| {
+                let node_id = Pubkey::new_rand();
+                (
+                    node_id,
+                    iter::repeat_with(|| {
+                        let authorized_voter = Pubkey::new_rand();
+                        VoteAccountInfo {
+                            vote_account: Pubkey::new_rand(),
+                            account: create_account_with_authorized(
+                                &node_id,
+                                &authorized_voter,
+                                &node_id,
+                                0,
+                                100,
+                            ),
+                            authorized_voter,
+                        }
+                    })
+                    .take(num_vote_accounts_per_node)
+                    .collect(),
+                )
+            })
+            .collect();
+
+        let expected_authorized_voters: HashMap<_, _> = vote_accounts_map
+            .iter()
+            .flat_map(|(_, vote_accounts)| {
+                vote_accounts
+                    .iter()
+                    .map(|v| (v.vote_account, v.authorized_voter))
+            })
+            .collect();
+
+        let expected_node_id_to_vote_accounts: HashMap<_, _> = vote_accounts_map
+            .iter()
+            .map(|(node_pubkey, vote_accounts)| {
+                let mut vote_accounts = vote_accounts
+                    .iter()
+                    .map(|v| (v.vote_account))
+                    .collect::<Vec<_>>();
+                vote_accounts.sort();
+                let node_vote_accounts = NodeVoteAccounts {
+                    vote_accounts,
+                    total_stake: stake_per_account * num_vote_accounts_per_node as u64,
+                };
+                (*node_pubkey, node_vote_accounts)
+            })
+            .collect();
+
+        // Create and process the vote accounts
+        let epoch_vote_accounts: HashMap<_, _> = vote_accounts_map
+            .iter()
+            .flat_map(|(_, vote_accounts)| {
+                vote_accounts
+                    .iter()
+                    .map(|v| (v.vote_account, (stake_per_account, v.account.clone())))
+            })
+            .collect();
+
+        let (total_stake, mut node_id_to_vote_accounts, epoch_authorized_voters) =
+            EpochStakes::parse_epoch_vote_accounts(&epoch_vote_accounts, 0);
+
+        // Verify the results
+        node_id_to_vote_accounts
+            .iter_mut()
+            .for_each(|(_, node_vote_accounts)| node_vote_accounts.vote_accounts.sort());
+
+        assert!(
+            node_id_to_vote_accounts.len() == expected_node_id_to_vote_accounts.len()
+                && node_id_to_vote_accounts
+                    .iter()
+                    .all(|(k, v)| expected_node_id_to_vote_accounts.get(k).unwrap() == v)
+        );
+        assert!(
+            epoch_authorized_voters.len() == expected_authorized_voters.len()
+                && epoch_authorized_voters
+                    .iter()
+                    .all(|(k, v)| expected_authorized_voters.get(k).unwrap() == v)
+        );
+        assert_eq!(
+            total_stake,
+            vote_accounts_map.len() as u64 * num_vote_accounts_per_node as u64 * 100
+        );
+    }
+}

runtime/src/lib.rs

@@ -6,6 +6,7 @@ pub mod bank;
 pub mod bank_client;
 mod blockhash_queue;
 pub mod bloom;
+pub mod epoch_stakes;
 pub mod genesis_utils;
 pub mod loader_utils;
 pub mod message_processor;