Extract process_ledger from Bank

Fullnode was the only real consumer of process_ledger and it was
only there to process a Blocktree. Blocktree is a tree, and a
ledger is a sequence, so something's clearly not right here.
Drop all other dependencies on process_ledger (only one test) so
that it can be fixed up in isolation.

src/bank.rs

@@ -6,7 +6,6 @@
 use crate::accounts::{Accounts, ErrorCounters, InstructionAccounts, InstructionLoaders};
 use crate::counter::Counter;
 use crate::entry::Entry;
-use crate::entry::EntrySlice;
 use crate::genesis_block::GenesisBlock;
 use crate::last_id_queue::{LastIdQueue, MAX_ENTRY_IDS};
 use crate::leader_scheduler::{LeaderScheduler, LeaderSchedulerConfig};
@@ -15,7 +14,6 @@ use crate::result::Error;
 use crate::rpc_pubsub::RpcSubscriptions;
 use crate::status_cache::StatusCache;
 use bincode::deserialize;
-use itertools::Itertools;
 use log::Level;
 use rayon::prelude::*;
 use solana_runtime::{self, RuntimeError};
@@ -87,8 +85,6 @@ pub enum BankError {
 
 pub type Result<T> = result::Result<T, BankError>;
 
-pub const VERIFY_BLOCK_SIZE: usize = 16;
-
 pub trait BankSubscriptions {
     fn check_account(&self, pubkey: &Pubkey, account: &Account);
     fn check_signature(&self, signature: &Signature, status: &Result<()>);
@@ -740,58 +736,6 @@ impl Bank {
         Ok(())
     }
 
-    /// Process an ordered list of entries, populating a circular buffer "tail"
-    /// as we go.
-    fn process_block(&self, entries: &[Entry]) -> Result<()> {
-        for entry in entries {
-            self.process_entry(entry)?;
-        }
-
-        Ok(())
-    }
-
-    /// Starting from the genesis block, append the provided entries to the ledger verifying them
-    /// along the way.
-    pub fn process_ledger<I>(&mut self, entries: I) -> Result<(u64, Hash)>
-    where
-        I: IntoIterator<Item = Entry>,
-    {
-        let mut last_entry_id = self.last_id();
-        let mut entries_iter = entries.into_iter();
-
-        trace!("genesis last_id={}", last_entry_id);
-
-        // The first entry in the ledger is a pseudo-tick used only to ensure the number of ticks
-        // in slot 0 is the same as the number of ticks in all subsequent slots.  It is not
-        // registered as a tick and thus cannot be used as a last_id
-        let entry0 = entries_iter
-            .next()
-            .ok_or(BankError::LedgerVerificationFailed)?;
-        if !(entry0.is_tick() && entry0.verify(&last_entry_id)) {
-            warn!("Ledger proof of history failed at entry0");
-            return Err(BankError::LedgerVerificationFailed);
-        }
-        last_entry_id = entry0.id;
-        let mut entry_height = 1;
-
-        // Ledger verification needs to be parallelized, but we can't pull the whole
-        // thing into memory. We therefore chunk it.
-        for block in &entries_iter.chunks(VERIFY_BLOCK_SIZE) {
-            let block: Vec<_> = block.collect();
-
-            if !block.verify(&last_entry_id) {
-                warn!("Ledger proof of history failed at entry: {}", entry_height);
-                return Err(BankError::LedgerVerificationFailed);
-            }
-
-            self.process_block(&block)?;
-
-            last_entry_id = block.last().unwrap().id;
-            entry_height += block.len() as u64;
-        }
-        Ok((entry_height, last_entry_id))
-    }
-
     /// Create, sign, and process a Transaction from `keypair` to `to` of
     /// `n` tokens where `last_id` is the last Entry ID observed by the client.
     pub fn transfer(
@@ -1197,7 +1141,7 @@ mod tests {
         genesis_block: &GenesisBlock,
         mint_keypair: &Keypair,
         keypairs: &[Keypair],
-    ) -> impl Iterator<Item = Entry> {
+    ) -> Vec<Entry> {
         let mut entries: Vec<Entry> = vec![];
 
         let mut last_id = genesis_block.last_id();
@@ -1220,83 +1164,7 @@ mod tests {
             last_id = hash;
             entries.push(tick);
         }
-        entries.into_iter()
+        entries
-    }
-
-    // create a ledger with a tick every `tick_interval` entries and a couple other transactions
-    fn create_sample_block_with_ticks(
-        genesis_block: &GenesisBlock,
-        mint_keypair: &Keypair,
-        num_one_token_transfers: usize,
-        tick_interval: usize,
-    ) -> impl Iterator<Item = Entry> {
-        let mut entries = vec![];
-
-        let mut last_id = genesis_block.last_id();
-
-        // Start off the ledger with the psuedo-tick linked to the genesis block
-        // (see entry0 in `process_ledger`)
-        let tick = Entry::new(&genesis_block.last_id(), 0, 1, vec![]);
-        let mut hash = tick.id;
-        entries.push(tick);
-
-        for i in 0..num_one_token_transfers {
-            // Transfer one token from the mint to a random account
-            let keypair = Keypair::new();
-            let tx = SystemTransaction::new_account(mint_keypair, keypair.pubkey(), 1, last_id, 0);
-            let entry = Entry::new(&hash, 0, 1, vec![tx]);
-            hash = entry.id;
-            entries.push(entry);
-
-            // Add a second Transaction that will produce a
-            // ProgramError<0, ResultWithNegativeTokens> error when processed
-            let keypair2 = Keypair::new();
-            let tx = SystemTransaction::new_account(&keypair, keypair2.pubkey(), 42, last_id, 0);
-            let entry = Entry::new(&hash, 0, 1, vec![tx]);
-            hash = entry.id;
-            entries.push(entry);
-
-            if (i + 1) % tick_interval == 0 {
-                let tick = Entry::new(&hash, 0, 1, vec![]);
-                hash = tick.id;
-                last_id = hash;
-                entries.push(tick);
-            }
-        }
-        entries.into_iter()
-    }
-
-    fn create_sample_ledger(
-        tokens: u64,
-        num_one_token_transfers: usize,
-    ) -> (GenesisBlock, Keypair, impl Iterator<Item = Entry>) {
-        let (genesis_block, mint_keypair) = GenesisBlock::new(tokens);
-        let block = create_sample_block_with_ticks(
-            &genesis_block,
-            &mint_keypair,
-            num_one_token_transfers,
-            num_one_token_transfers,
-        );
-        (genesis_block, mint_keypair, block)
-    }
-
-    #[test]
-    fn test_process_ledger_simple() {
-        let (genesis_block, mint_keypair, ledger) = create_sample_ledger(100, 3);
-        let mut bank = Bank::default();
-        bank.add_builtin_programs();
-        bank.process_genesis_block(&genesis_block);
-        assert_eq!(bank.tick_height(), 0);
-        assert_eq!(bank.get_balance(&mint_keypair.pubkey()), 100);
-        assert_eq!(
-            bank.get_current_leader(),
-            Some(genesis_block.bootstrap_leader_id)
-        );
-        let (ledger_height, last_id) = bank.process_ledger(ledger).unwrap();
-        assert_eq!(bank.get_balance(&mint_keypair.pubkey()), 100 - 3);
-        assert_eq!(ledger_height, 8);
-        assert_eq!(bank.tick_height(), 1);
-        assert_eq!(bank.last_id(), last_id);
     }
 
     #[test]
@@ -1305,25 +1173,25 @@ mod tests {
         let seed = [0u8; 32];
         let mut rnd = GenKeys::new(seed);
         let keypairs = rnd.gen_n_keypairs(5);
-        let ledger0 = create_sample_block_with_next_entries_using_keypairs(
+        let entries0 = create_sample_block_with_next_entries_using_keypairs(
             &genesis_block,
             &mint_keypair,
             &keypairs,
         );
-        let ledger1 = create_sample_block_with_next_entries_using_keypairs(
+        let entries1 = create_sample_block_with_next_entries_using_keypairs(
             &genesis_block,
             &mint_keypair,
             &keypairs,
         );
 
-        let mut bank0 = Bank::default();
+        let bank0 = Bank::default();
         bank0.add_builtin_programs();
         bank0.process_genesis_block(&genesis_block);
-        bank0.process_ledger(ledger0).unwrap();
+        bank0.process_entries(&entries0).unwrap();
-        let mut bank1 = Bank::default();
+        let bank1 = Bank::default();
         bank1.add_builtin_programs();
         bank1.process_genesis_block(&genesis_block);
-        bank1.process_ledger(ledger1).unwrap();
+        bank1.process_entries(&entries1).unwrap();
 
         let initial_state = bank0.hash_internal_state();
 

src/blocktree_processor.rs

@@ -0,0 +1,142 @@
+use crate::bank::{Bank, BankError, Result};
+use crate::blocktree::Blocktree;
+use crate::entry::{Entry, EntrySlice};
+use itertools::Itertools;
+use solana_sdk::hash::Hash;
+
+pub const VERIFY_BLOCK_SIZE: usize = 16;
+
+/// Process an ordered list of entries, populating a circular buffer "tail"
+/// as we go.
+fn process_block(bank: &Bank, entries: &[Entry]) -> Result<()> {
+    for entry in entries {
+        bank.process_entry(entry)?;
+    }
+
+    Ok(())
+}
+
+/// Starting from the genesis block, append the provided entries to the ledger verifying them
+/// along the way.
+fn process_ledger<I>(bank: &Bank, entries: I) -> Result<(u64, Hash)>
+where
+    I: IntoIterator<Item = Entry>,
+{
+    let mut last_entry_id = bank.last_id();
+    let mut entries_iter = entries.into_iter();
+
+    trace!("genesis last_id={}", last_entry_id);
+
+    // The first entry in the ledger is a pseudo-tick used only to ensure the number of ticks
+    // in slot 0 is the same as the number of ticks in all subsequent slots.  It is not
+    // registered as a tick and thus cannot be used as a last_id
+    let entry0 = entries_iter
+        .next()
+        .ok_or(BankError::LedgerVerificationFailed)?;
+    if !(entry0.is_tick() && entry0.verify(&last_entry_id)) {
+        warn!("Ledger proof of history failed at entry0");
+        return Err(BankError::LedgerVerificationFailed);
+    }
+    last_entry_id = entry0.id;
+    let mut entry_height = 1;
+
+    // Ledger verification needs to be parallelized, but we can't pull the whole
+    // thing into memory. We therefore chunk it.
+    for block in &entries_iter.chunks(VERIFY_BLOCK_SIZE) {
+        let block: Vec<_> = block.collect();
+
+        if !block.verify(&last_entry_id) {
+            warn!("Ledger proof of history failed at entry: {}", entry_height);
+            return Err(BankError::LedgerVerificationFailed);
+        }
+
+        process_block(bank, &block)?;
+
+        last_entry_id = block.last().unwrap().id;
+        entry_height += block.len() as u64;
+    }
+    Ok((entry_height, last_entry_id))
+}
+
+pub fn process_blocktree(bank: &Bank, blocktree: &Blocktree) -> Result<(u64, Hash)> {
+    let entries = blocktree.read_ledger().expect("opening ledger");
+    process_ledger(&bank, entries)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::genesis_block::GenesisBlock;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::system_transaction::SystemTransaction;
+
+    // create a ledger with a tick every `tick_interval` entries and a couple other transactions
+    fn create_sample_block_with_ticks(
+        genesis_block: &GenesisBlock,
+        mint_keypair: &Keypair,
+        num_one_token_transfers: usize,
+        tick_interval: usize,
+    ) -> impl Iterator<Item = Entry> {
+        let mut entries = vec![];
+
+        let mut last_id = genesis_block.last_id();
+
+        // Start off the ledger with the psuedo-tick linked to the genesis block
+        // (see entry0 in `process_ledger`)
+        let tick = Entry::new(&genesis_block.last_id(), 0, 1, vec![]);
+        let mut hash = tick.id;
+        entries.push(tick);
+
+        for i in 0..num_one_token_transfers {
+            // Transfer one token from the mint to a random account
+            let keypair = Keypair::new();
+            let tx = SystemTransaction::new_account(mint_keypair, keypair.pubkey(), 1, last_id, 0);
+            let entry = Entry::new(&hash, 0, 1, vec![tx]);
+            hash = entry.id;
+            entries.push(entry);
+
+            // Add a second Transaction that will produce a
+            // ProgramError<0, ResultWithNegativeTokens> error when processed
+            let keypair2 = Keypair::new();
+            let tx = SystemTransaction::new_account(&keypair, keypair2.pubkey(), 42, last_id, 0);
+            let entry = Entry::new(&hash, 0, 1, vec![tx]);
+            hash = entry.id;
+            entries.push(entry);
+
+            if (i + 1) % tick_interval == 0 {
+                let tick = Entry::new(&hash, 0, 1, vec![]);
+                hash = tick.id;
+                last_id = hash;
+                entries.push(tick);
+            }
+        }
+        entries.into_iter()
+    }
+
+    fn create_sample_ledger(
+        tokens: u64,
+        num_one_token_transfers: usize,
+    ) -> (GenesisBlock, Keypair, impl Iterator<Item = Entry>) {
+        let (genesis_block, mint_keypair) = GenesisBlock::new(tokens);
+        let block = create_sample_block_with_ticks(
+            &genesis_block,
+            &mint_keypair,
+            num_one_token_transfers,
+            num_one_token_transfers,
+        );
+        (genesis_block, mint_keypair, block)
+    }
+
+    #[test]
+    fn test_process_ledger_simple() {
+        let (genesis_block, mint_keypair, ledger) = create_sample_ledger(100, 3);
+        let bank = Bank::new(&genesis_block);
+        assert_eq!(bank.tick_height(), 0);
+        assert_eq!(bank.get_balance(&mint_keypair.pubkey()), 100);
+        let (ledger_height, last_id) = process_ledger(&bank, ledger).unwrap();
+        assert_eq!(bank.get_balance(&mint_keypair.pubkey()), 100 - 3);
+        assert_eq!(ledger_height, 8);
+        assert_eq!(bank.tick_height(), 1);
+        assert_eq!(bank.last_id(), last_id);
+    }
+}

src/fullnode.rs

@@ -2,6 +2,7 @@
 
 use crate::bank::Bank;
 use crate::blocktree::{Blocktree, BlocktreeConfig};
+use crate::blocktree_processor;
 use crate::cluster_info::{ClusterInfo, Node, NodeInfo};
 use crate::counter::Counter;
 use crate::genesis_block::GenesisBlock;
@@ -462,12 +463,12 @@ pub fn new_bank_from_ledger(
             .expect("Expected to successfully open database ledger");
     let genesis_block =
         GenesisBlock::load(ledger_path).expect("Expected to successfully open genesis block");
-    let mut bank = Bank::new_with_leader_scheduler_config(&genesis_block, leader_scheduler_config);
+    let bank = Bank::new_with_leader_scheduler_config(&genesis_block, leader_scheduler_config);
 
     let now = Instant::now();
-    let entries = blocktree.read_ledger().expect("opening ledger");
     info!("processing ledger...");
-    let (entry_height, last_entry_id) = bank.process_ledger(entries).expect("process_ledger");
+    let (entry_height, last_entry_id) =
+        blocktree_processor::process_blocktree(&bank, &blocktree).expect("process_blocktree");
     info!(
         "processed {} ledger entries in {}ms, tick_height={}...",
         entry_height,

src/lib.rs

@@ -30,6 +30,7 @@ pub mod crds_value;
 #[macro_use]
 pub mod contact_info;
 pub mod blocktree;
+pub mod blocktree_processor;
 pub mod cluster_info;
 pub mod compute_leader_confirmation_service;
 pub mod db_window;