Fix resetting PohRecorder to wrong bank (#3553)

* Check whether future slot already has transmission

core/benches/banking_stage.rs

@@ -1,10 +1,13 @@
 #![feature(test)]
 
 extern crate test;
+#[macro_use]
+extern crate solana;
 
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
 use solana::banking_stage::{create_test_recorder, BankingStage};
+use solana::blocktree::{get_tmp_ledger_path, Blocktree};
 use solana::cluster_info::ClusterInfo;
 use solana::cluster_info::Node;
 use solana::packet::to_packets_chunked;
@@ -104,7 +107,13 @@ fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) {
             (x, iter::repeat(1).take(len).collect())
         })
         .collect();
-    let (exit, poh_recorder, poh_service, signal_receiver) = create_test_recorder(&bank);
+    let ledger_path = get_tmp_ledger_path!();
+    {
+        let blocktree = Arc::new(
+            Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger"),
+        );
+        let (exit, poh_recorder, poh_service, signal_receiver) =
+            create_test_recorder(&bank, &blocktree);
         let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
         let cluster_info = Arc::new(RwLock::new(cluster_info));
         let _banking_stage = BankingStage::new(&cluster_info, &poh_recorder, verified_receiver);
@@ -131,6 +140,8 @@ fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) {
         });
         exit.store(true, Ordering::Relaxed);
         poh_service.join().unwrap();
+    }
+    Blocktree::destroy(&ledger_path).unwrap();
 }
 
 #[bench]
@@ -211,7 +222,14 @@ fn bench_banking_stage_multi_programs(bencher: &mut Bencher) {
             (x, iter::repeat(1).take(len).collect())
         })
         .collect();
-    let (exit, poh_recorder, poh_service, signal_receiver) = create_test_recorder(&bank);
+
+    let ledger_path = get_tmp_ledger_path!();
+    {
+        let blocktree = Arc::new(
+            Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger"),
+        );
+        let (exit, poh_recorder, poh_service, signal_receiver) =
+            create_test_recorder(&bank, &blocktree);
         let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
         let cluster_info = Arc::new(RwLock::new(cluster_info));
         let _banking_stage = BankingStage::new(&cluster_info, &poh_recorder, verified_receiver);
@@ -238,4 +256,6 @@ fn bench_banking_stage_multi_programs(bencher: &mut Bencher) {
         });
         exit.store(true, Ordering::Relaxed);
         poh_service.join().unwrap();
+    }
+    Blocktree::destroy(&ledger_path).unwrap();
 }

core/src/banking_stage.rs

@@ -1,7 +1,7 @@
 //! The `banking_stage` processes Transaction messages. It is intended to be used
 //! to contruct a software pipeline. The stage uses all available CPU cores and
 //! can do its processing in parallel with signature verification on the GPU.
-
+use crate::blocktree::Blocktree;
 use crate::cluster_info::ClusterInfo;
 use crate::entry;
 use crate::entry::{hash_transactions, Entry};
@@ -472,6 +472,7 @@ impl Service for BankingStage {
 
 pub fn create_test_recorder(
     bank: &Arc<Bank>,
+    blocktree: &Arc<Blocktree>,
 ) -> (
     Arc<AtomicBool>,
     Arc<Mutex<PohRecorder>>,
@@ -486,6 +487,7 @@ pub fn create_test_recorder(
         Some(4),
         bank.ticks_per_slot(),
         &Pubkey::default(),
+        blocktree,
     );
     poh_recorder.set_bank(&bank);
 
@@ -498,10 +500,12 @@ pub fn create_test_recorder(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::blocktree::get_tmp_ledger_path;
     use crate::cluster_info::Node;
     use crate::entry::EntrySlice;
     use crate::packet::to_packets;
     use crate::poh_recorder::WorkingBank;
+    use crate::{get_tmp_ledger_path, tmp_ledger_name};
     use solana_sdk::genesis_block::GenesisBlock;
     use solana_sdk::instruction::InstructionError;
     use solana_sdk::signature::{Keypair, KeypairUtil};
@@ -514,7 +518,13 @@ mod tests {
         let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
         let bank = Arc::new(Bank::new(&genesis_block));
         let (verified_sender, verified_receiver) = channel();
-        let (exit, poh_recorder, poh_service, _entry_receiever) = create_test_recorder(&bank);
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree = Arc::new(
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger"),
+            );
+            let (exit, poh_recorder, poh_service, _entry_receiever) =
+                create_test_recorder(&bank, &blocktree);
             let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
             let cluster_info = Arc::new(RwLock::new(cluster_info));
             let banking_stage = BankingStage::new(&cluster_info, &poh_recorder, verified_receiver);
@@ -523,6 +533,8 @@ mod tests {
             banking_stage.join().unwrap();
             poh_service.join().unwrap();
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_banking_stage_tick() {
@@ -532,7 +544,13 @@ mod tests {
         let bank = Arc::new(Bank::new(&genesis_block));
         let start_hash = bank.last_blockhash();
         let (verified_sender, verified_receiver) = channel();
-        let (exit, poh_recorder, poh_service, entry_receiver) = create_test_recorder(&bank);
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree = Arc::new(
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger"),
+            );
+            let (exit, poh_recorder, poh_service, entry_receiver) =
+                create_test_recorder(&bank, &blocktree);
             let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
             let cluster_info = Arc::new(RwLock::new(cluster_info));
             let banking_stage = BankingStage::new(&cluster_info, &poh_recorder, verified_receiver);
@@ -554,6 +572,8 @@ mod tests {
             assert_eq!(entries[entries.len() - 1].hash, bank.last_blockhash());
             banking_stage.join().unwrap();
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_banking_stage_entries_only() {
@@ -562,7 +582,13 @@ mod tests {
         let bank = Arc::new(Bank::new(&genesis_block));
         let start_hash = bank.last_blockhash();
         let (verified_sender, verified_receiver) = channel();
-        let (exit, poh_recorder, poh_service, entry_receiver) = create_test_recorder(&bank);
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree = Arc::new(
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger"),
+            );
+            let (exit, poh_recorder, poh_service, entry_receiver) =
+                create_test_recorder(&bank, &blocktree);
             let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
             let cluster_info = Arc::new(RwLock::new(cluster_info));
             let banking_stage = BankingStage::new(&cluster_info, &poh_recorder, verified_receiver);
@@ -633,6 +659,8 @@ mod tests {
             drop(entry_receiver);
             banking_stage.join().unwrap();
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_banking_stage_entryfication() {
@@ -671,14 +699,26 @@ mod tests {
             .send(vec![(packets[0].clone(), vec![1u8])])
             .unwrap();
 
+        let ledger_path = get_tmp_ledger_path!();
+        {
             let entry_receiver = {
                 // start a banking_stage to eat verified receiver
                 let bank = Arc::new(Bank::new(&genesis_block));
-            let (exit, poh_recorder, poh_service, entry_receiver) = create_test_recorder(&bank);
-            let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
+                let blocktree = Arc::new(
+                    Blocktree::open(&ledger_path)
+                        .expect("Expected to be able to open database ledger"),
+                );
+                let (exit, poh_recorder, poh_service, entry_receiver) =
+                    create_test_recorder(&bank, &blocktree);
+                let cluster_info =
+                    ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
                 let cluster_info = Arc::new(RwLock::new(cluster_info));
-            let _banking_stage =
-                BankingStage::new_num_threads(&cluster_info, &poh_recorder, verified_receiver, 1);
+                let _banking_stage = BankingStage::new_num_threads(
+                    &cluster_info,
+                    &poh_recorder,
+                    verified_receiver,
+                    1,
+                );
 
                 // wait for banking_stage to eat the packets
                 while bank.get_balance(&alice.pubkey()) != 1 {
@@ -709,6 +749,8 @@ mod tests {
             // the account balance below zero before the credit is added.
             assert_eq!(bank.get_balance(&alice.pubkey()), 1);
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_bank_record_transactions() {
@@ -719,7 +761,10 @@ mod tests {
             min_tick_height: bank.tick_height(),
             max_tick_height: std::u64::MAX,
         };
-
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (poh_recorder, entry_receiver) = PohRecorder::new(
                 bank.tick_height(),
                 bank.last_blockhash(),
@@ -727,6 +772,7 @@ mod tests {
                 None,
                 bank.ticks_per_slot(),
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
             let poh_recorder = Arc::new(Mutex::new(poh_recorder));
 
@@ -761,6 +807,8 @@ mod tests {
             let (_, entries) = entry_receiver.recv().unwrap();
             assert_eq!(entries[0].0.transactions.len(), transactions.len() - 1);
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_bank_process_and_record_transactions() {
@@ -782,6 +830,10 @@ mod tests {
             min_tick_height: bank.tick_height(),
             max_tick_height: bank.tick_height() + 1,
         };
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (poh_recorder, entry_receiver) = PohRecorder::new(
                 bank.tick_height(),
                 bank.last_blockhash(),
@@ -789,12 +841,14 @@ mod tests {
                 Some(4),
                 bank.ticks_per_slot(),
                 &pubkey,
+                &Arc::new(blocktree),
             );
             let poh_recorder = Arc::new(Mutex::new(poh_recorder));
 
             poh_recorder.lock().unwrap().set_working_bank(working_bank);
 
-        BankingStage::process_and_record_transactions(&bank, &transactions, &poh_recorder).unwrap();
+            BankingStage::process_and_record_transactions(&bank, &transactions, &poh_recorder)
+                .unwrap();
             poh_recorder.lock().unwrap().tick();
 
             let mut done = false;
@@ -831,4 +885,6 @@ mod tests {
 
             assert_eq!(bank.get_balance(&pubkey), 1);
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 }

core/src/fullnode.rs

@@ -105,13 +105,15 @@ impl Fullnode {
             bank.tick_height(),
             bank.last_blockhash(),
         );
+        let blocktree = Arc::new(blocktree);
         let (poh_recorder, entry_receiver) = PohRecorder::new(
             bank.tick_height(),
             bank.last_blockhash(),
             bank.slot(),
-            leader_schedule_utils::next_leader_slot(&id, bank.slot(), &bank),
+            leader_schedule_utils::next_leader_slot(&id, bank.slot(), &bank, Some(&blocktree)),
             bank.ticks_per_slot(),
             &id,
+            &blocktree,
         );
         let poh_recorder = Arc::new(Mutex::new(poh_recorder));
         let poh_service = PohService::new(poh_recorder.clone(), &config.tick_config, &exit);
@@ -130,7 +132,6 @@ impl Fullnode {
             node.sockets.gossip.local_addr().unwrap()
         );
 
-        let blocktree = Arc::new(blocktree);
         let bank_forks = Arc::new(RwLock::new(bank_forks));
 
         node.info.wallclock = timestamp();

core/src/leader_schedule_utils.rs

@@ -1,3 +1,4 @@
+use crate::blocktree::Blocktree;
 use crate::leader_schedule::LeaderSchedule;
 use crate::staking_utils;
 use solana_runtime::bank::Bank;
@@ -44,7 +45,12 @@ pub fn slot_leader_at(slot: u64, bank: &Bank) -> Option<Pubkey> {
 }
 
 /// Return the next slot after the given current_slot that the given node will be leader
-pub fn next_leader_slot(pubkey: &Pubkey, mut current_slot: u64, bank: &Bank) -> Option<u64> {
+pub fn next_leader_slot(
+    pubkey: &Pubkey,
+    mut current_slot: u64,
+    bank: &Bank,
+    blocktree: Option<&Blocktree>,
+) -> Option<u64> {
     let (mut epoch, mut start_index) = bank.get_epoch_and_slot_index(current_slot + 1);
     while let Some(leader_schedule) = leader_schedule(epoch, bank) {
         // clippy thinks I should do this:
@@ -59,6 +65,15 @@ pub fn next_leader_slot(pubkey: &Pubkey, mut current_slot: u64, bank: &Bank) ->
         for i in start_index..bank.get_slots_in_epoch(epoch) {
             current_slot += 1;
             if *pubkey == leader_schedule[i] {
+                if let Some(blocktree) = blocktree {
+                    if let Some(meta) = blocktree.meta(current_slot).unwrap() {
+                        // We have already sent a blob for this slot, so skip it
+                        if meta.received > 0 {
+                            continue;
+                        }
+                    }
+                }
+
                 return Some(current_slot);
             }
         }
@@ -82,6 +97,8 @@ pub fn tick_height_to_slot(ticks_per_slot: u64, tick_height: u64) -> u64 {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::blocktree::get_tmp_ledger_path;
+    use crate::blocktree::tests::make_slot_entries;
     use crate::staking_utils;
     use crate::voting_keypair::tests::new_vote_account_with_delegate;
     use solana_sdk::genesis_block::{GenesisBlock, BOOTSTRAP_LEADER_LAMPORTS};
@@ -101,13 +118,14 @@ mod tests {
 
         let bank = Bank::new(&genesis_block);
         assert_eq!(slot_leader_at(bank.slot(), &bank).unwrap(), pubkey);
-        assert_eq!(next_leader_slot(&pubkey, 0, &bank), Some(1));
-        assert_eq!(next_leader_slot(&pubkey, 1, &bank), Some(2));
+        assert_eq!(next_leader_slot(&pubkey, 0, &bank, None), Some(1));
+        assert_eq!(next_leader_slot(&pubkey, 1, &bank, None), Some(2));
         assert_eq!(
             next_leader_slot(
                 &pubkey,
                 2 * genesis_block.slots_per_epoch - 1, // no schedule generated for epoch 2
-                &bank
+                &bank,
+                None
             ),
             None
         );
@@ -116,12 +134,81 @@ mod tests {
             next_leader_slot(
                 &Keypair::new().pubkey(), // not in leader_schedule
                 0,
-                &bank
+                &bank,
+                None
             ),
             None
         );
     }
 
+    #[test]
+    fn test_next_leader_slot_blocktree() {
+        let pubkey = Keypair::new().pubkey();
+        let mut genesis_block = GenesisBlock::new_with_leader(
+            BOOTSTRAP_LEADER_LAMPORTS,
+            &pubkey,
+            BOOTSTRAP_LEADER_LAMPORTS,
+        )
+        .0;
+        genesis_block.epoch_warmup = false;
+
+        let bank = Bank::new(&genesis_block);
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree = Arc::new(
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger"),
+            );
+
+            assert_eq!(slot_leader_at(bank.slot(), &bank).unwrap(), pubkey);
+            // Check that the next leader slot after 0 is slot 1
+            assert_eq!(
+                next_leader_slot(&pubkey, 0, &bank, Some(&blocktree)),
+                Some(1)
+            );
+
+            // Write a blob into slot 2 that chains to slot 1,
+            // but slot 1 is empty so should not be skipped
+            let (blobs, _) = make_slot_entries(2, 1, 1);
+            blocktree.write_blobs(&blobs[..]).unwrap();
+            assert_eq!(
+                next_leader_slot(&pubkey, 0, &bank, Some(&blocktree)),
+                Some(1)
+            );
+
+            // Write a blob into slot 1
+            let (blobs, _) = make_slot_entries(1, 0, 1);
+
+            // Check that slot 1 and 2 are skipped
+            blocktree.write_blobs(&blobs[..]).unwrap();
+            assert_eq!(
+                next_leader_slot(&pubkey, 0, &bank, Some(&blocktree)),
+                Some(3)
+            );
+
+            // Integrity checks
+            assert_eq!(
+                next_leader_slot(
+                    &pubkey,
+                    2 * genesis_block.slots_per_epoch - 1, // no schedule generated for epoch 2
+                    &bank,
+                    Some(&blocktree)
+                ),
+                None
+            );
+
+            assert_eq!(
+                next_leader_slot(
+                    &Keypair::new().pubkey(), // not in leader_schedule
+                    0,
+                    &bank,
+                    Some(&blocktree)
+                ),
+                None
+            );
+        }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
+
     #[test]
     fn test_next_leader_slot_next_epoch() {
         let pubkey = Keypair::new().pubkey();
@@ -169,8 +256,8 @@ mod tests {
         expected_slot += index;
 
         assert_eq!(
-            next_leader_slot(&delegate_id, 0, &bank),
-            Some(expected_slot)
+            next_leader_slot(&delegate_id, 0, &bank, None),
+            Some(expected_slot),
         );
     }
 

core/src/poh_recorder.rs

@@ -10,6 +10,7 @@
 //! For Entries:
 //! * recorded entry must be >= WorkingBank::min_tick_height && entry must be < WorkingBank::man_tick_height
 //!
+use crate::blocktree::Blocktree;
 use crate::entry::Entry;
 use crate::leader_schedule_utils;
 use crate::poh::Poh;
@@ -51,14 +52,19 @@ pub struct PohRecorder {
     last_leader_tick: Option<u64>,
     max_last_leader_grace_ticks: u64,
     id: Pubkey,
+    blocktree: Arc<Blocktree>,
 }
 
 impl PohRecorder {
     pub fn clear_bank(&mut self) {
         if let Some(working_bank) = self.working_bank.take() {
             let bank = working_bank.bank;
-            let next_leader_slot =
-                leader_schedule_utils::next_leader_slot(&self.id, bank.slot(), &bank);
+            let next_leader_slot = leader_schedule_utils::next_leader_slot(
+                &self.id,
+                bank.slot(),
+                &bank,
+                Some(&self.blocktree),
+            );
             let (start_leader_at_tick, last_leader_tick) = Self::compute_leader_slot_ticks(
                 &next_leader_slot,
                 bank.ticks_per_slot(),
@@ -273,6 +279,7 @@ impl PohRecorder {
         my_leader_slot_index: Option<u64>,
         ticks_per_slot: u64,
         id: &Pubkey,
+        blocktree: &Arc<Blocktree>,
     ) -> (Self, Receiver<WorkingBankEntries>) {
         let poh = Poh::new(last_entry_hash, tick_height);
         let (sender, receiver) = channel();
@@ -295,6 +302,7 @@ impl PohRecorder {
                 last_leader_tick,
                 max_last_leader_grace_ticks,
                 id: *id,
+                blocktree: blocktree.clone(),
             },
             receiver,
         )
@@ -345,6 +353,7 @@ impl PohRecorder {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::blocktree::{get_tmp_ledger_path, Blocktree};
     use crate::test_tx::test_tx;
     use solana_sdk::genesis_block::GenesisBlock;
     use solana_sdk::hash::hash;
@@ -355,6 +364,11 @@ mod tests {
     #[test]
     fn test_poh_recorder_no_zero_tick() {
         let prev_hash = Hash::default();
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
+
             let (mut poh_recorder, _entry_receiver) = PohRecorder::new(
                 0,
                 prev_hash,
@@ -362,16 +376,24 @@ mod tests {
                 Some(4),
                 DEFAULT_TICKS_PER_SLOT,
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
             poh_recorder.tick();
             assert_eq!(poh_recorder.tick_cache.len(), 1);
             assert_eq!(poh_recorder.tick_cache[0].1, 1);
             assert_eq!(poh_recorder.poh.tick_height, 1);
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_poh_recorder_tick_height_is_last_tick() {
         let prev_hash = Hash::default();
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
+
             let (mut poh_recorder, _entry_receiver) = PohRecorder::new(
                 0,
                 prev_hash,
@@ -379,6 +401,7 @@ mod tests {
                 Some(4),
                 DEFAULT_TICKS_PER_SLOT,
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
             poh_recorder.tick();
             poh_recorder.tick();
@@ -386,9 +409,15 @@ mod tests {
             assert_eq!(poh_recorder.tick_cache[1].1, 2);
             assert_eq!(poh_recorder.poh.tick_height, 2);
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_poh_recorder_reset_clears_cache() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (mut poh_recorder, _entry_receiver) = PohRecorder::new(
                 0,
                 Hash::default(),
@@ -396,15 +425,22 @@ mod tests {
                 Some(4),
                 DEFAULT_TICKS_PER_SLOT,
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
             poh_recorder.tick();
             assert_eq!(poh_recorder.tick_cache.len(), 1);
             poh_recorder.reset(0, Hash::default(), 0, Some(4), DEFAULT_TICKS_PER_SLOT);
             assert_eq!(poh_recorder.tick_cache.len(), 0);
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_poh_recorder_clear() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
             let bank = Arc::new(Bank::new(&genesis_block));
             let prev_hash = bank.last_blockhash();
@@ -415,6 +451,7 @@ mod tests {
                 Some(4),
                 bank.ticks_per_slot(),
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
 
             let working_bank = WorkingBank {
@@ -427,9 +464,15 @@ mod tests {
             poh_recorder.clear_bank();
             assert!(poh_recorder.working_bank.is_none());
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_poh_recorder_tick_sent_after_min() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
             let bank = Arc::new(Bank::new(&genesis_block));
             let prev_hash = bank.last_blockhash();
@@ -440,6 +483,7 @@ mod tests {
                 Some(4),
                 bank.ticks_per_slot(),
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
 
             let working_bank = WorkingBank {
@@ -464,9 +508,15 @@ mod tests {
             assert_eq!(bank_.slot(), bank.slot());
             assert!(poh_recorder.working_bank.is_none());
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_poh_recorder_tick_sent_upto_and_including_max() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
             let bank = Arc::new(Bank::new(&genesis_block));
             let prev_hash = bank.last_blockhash();
@@ -477,6 +527,7 @@ mod tests {
                 Some(4),
                 bank.ticks_per_slot(),
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
 
             poh_recorder.tick();
@@ -499,9 +550,15 @@ mod tests {
             let (_, e) = entry_receiver.recv().expect("recv 1");
             assert_eq!(e.len(), 3);
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_poh_recorder_record_to_early() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
             let bank = Arc::new(Bank::new(&genesis_block));
             let prev_hash = bank.last_blockhash();
@@ -512,6 +569,7 @@ mod tests {
                 Some(4),
                 bank.ticks_per_slot(),
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
 
             let working_bank = WorkingBank {
@@ -528,9 +586,15 @@ mod tests {
                 .is_err());
             assert!(entry_receiver.try_recv().is_err());
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_poh_recorder_record_bad_slot() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
             let bank = Arc::new(Bank::new(&genesis_block));
             let prev_hash = bank.last_blockhash();
@@ -541,6 +605,7 @@ mod tests {
                 Some(4),
                 bank.ticks_per_slot(),
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
 
             let working_bank = WorkingBank {
@@ -559,9 +624,15 @@ mod tests {
                 Err(Error::PohRecorderError(PohRecorderError::MaxHeightReached))
             );
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_poh_recorder_record_at_min_passes() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
             let bank = Arc::new(Bank::new(&genesis_block));
             let prev_hash = bank.last_blockhash();
@@ -572,6 +643,7 @@ mod tests {
                 Some(4),
                 bank.ticks_per_slot(),
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
 
             let working_bank = WorkingBank {
@@ -597,9 +669,15 @@ mod tests {
             let (_b, e) = entry_receiver.recv().expect("recv 2");
             assert!(!e[0].0.is_tick());
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_poh_recorder_record_at_max_fails() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
             let bank = Arc::new(Bank::new(&genesis_block));
             let prev_hash = bank.last_blockhash();
@@ -610,6 +688,7 @@ mod tests {
                 Some(4),
                 bank.ticks_per_slot(),
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
 
             let working_bank = WorkingBank {
@@ -632,9 +711,15 @@ mod tests {
             assert!(e[0].0.is_tick());
             assert!(e[1].0.is_tick());
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_poh_cache_on_disconnect() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
             let bank = Arc::new(Bank::new(&genesis_block));
             let prev_hash = bank.last_blockhash();
@@ -645,6 +730,7 @@ mod tests {
                 Some(4),
                 bank.ticks_per_slot(),
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
 
             let working_bank = WorkingBank {
@@ -661,9 +747,15 @@ mod tests {
             assert!(poh_recorder.working_bank.is_none());
             assert_eq!(poh_recorder.tick_cache.len(), 3);
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_reset_current() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (mut poh_recorder, _entry_receiver) = PohRecorder::new(
                 0,
                 Hash::default(),
@@ -671,6 +763,7 @@ mod tests {
                 Some(4),
                 DEFAULT_TICKS_PER_SLOT,
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
             poh_recorder.tick();
             poh_recorder.tick();
@@ -684,9 +777,15 @@ mod tests {
             );
             assert_eq!(poh_recorder.tick_cache.len(), 0);
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_reset_with_cached() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (mut poh_recorder, _entry_receiver) = PohRecorder::new(
                 0,
                 Hash::default(),
@@ -694,6 +793,7 @@ mod tests {
                 Some(4),
                 DEFAULT_TICKS_PER_SLOT,
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
             poh_recorder.tick();
             poh_recorder.tick();
@@ -707,9 +807,15 @@ mod tests {
             );
             assert_eq!(poh_recorder.tick_cache.len(), 0);
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_reset_to_new_value() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (mut poh_recorder, _entry_receiver) = PohRecorder::new(
                 0,
                 Hash::default(),
@@ -717,6 +823,7 @@ mod tests {
                 Some(4),
                 DEFAULT_TICKS_PER_SLOT,
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
             poh_recorder.tick();
             poh_recorder.tick();
@@ -728,9 +835,15 @@ mod tests {
             poh_recorder.tick();
             assert_eq!(poh_recorder.poh.tick_height, 2);
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_reset_clear_bank() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
             let bank = Arc::new(Bank::new(&genesis_block));
             let (mut poh_recorder, _entry_receiver) = PohRecorder::new(
@@ -740,6 +853,7 @@ mod tests {
                 Some(4),
                 bank.ticks_per_slot(),
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
             let ticks_per_slot = bank.ticks_per_slot();
             let working_bank = WorkingBank {
@@ -751,9 +865,15 @@ mod tests {
             poh_recorder.reset(1, hash(b"hello"), 0, Some(4), ticks_per_slot);
             assert!(poh_recorder.working_bank.is_none());
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     pub fn test_clear_signal() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
             let bank = Arc::new(Bank::new(&genesis_block));
             let (mut poh_recorder, _entry_receiver) = PohRecorder::new(
@@ -763,6 +883,7 @@ mod tests {
                 None,
                 bank.ticks_per_slot(),
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
             let (sender, receiver) = sync_channel(1);
             poh_recorder.set_bank(&bank);
@@ -770,9 +891,15 @@ mod tests {
             poh_recorder.clear_bank();
             assert!(receiver.try_recv().is_ok());
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_poh_recorder_reset_start_slot() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let ticks_per_slot = 5;
             let (mut genesis_block, _mint_keypair) = GenesisBlock::new(2);
             genesis_block.ticks_per_slot = ticks_per_slot;
@@ -786,6 +913,7 @@ mod tests {
                 Some(4),
                 bank.ticks_per_slot(),
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
 
             let end_slot = 3;
@@ -810,9 +938,15 @@ mod tests {
             // Make sure the starting slot is updated
             assert_eq!(poh_recorder.start_slot(), end_slot);
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 
     #[test]
     fn test_reached_leader_tick() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
             let bank = Arc::new(Bank::new(&genesis_block));
             let prev_hash = bank.last_blockhash();
@@ -823,6 +957,7 @@ mod tests {
                 None,
                 bank.ticks_per_slot(),
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
 
             // Test that with no leader slot, we don't reach the leader tick
@@ -932,7 +1067,9 @@ mod tests {
             );
 
             // Send remaining ticks for the slot (remember we sent extra ticks in the previous part of the test)
-        for _ in bank.ticks_per_slot() / MAX_LAST_LEADER_GRACE_TICKS_FACTOR..bank.ticks_per_slot() {
+            for _ in
+                bank.ticks_per_slot() / MAX_LAST_LEADER_GRACE_TICKS_FACTOR..bank.ticks_per_slot()
+            {
                 poh_recorder.tick();
             }
 
@@ -971,4 +1108,6 @@ mod tests {
             // We are not the leader, as expected
             assert_eq!(poh_recorder.reached_leader_tick().0, false);
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 }

core/src/poh_service.rs

@@ -1,6 +1,5 @@
 //! The `poh_service` module implements a service that records the passing of
 //! "ticks", a measure of time in the PoH stream
-
 use crate::poh_recorder::PohRecorder;
 use crate::service::Service;
 use solana_sdk::timing::NUM_TICKS_PER_SECOND;
@@ -98,6 +97,7 @@ impl Service for PohService {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::blocktree::{get_tmp_ledger_path, Blocktree};
     use crate::poh_recorder::WorkingBank;
     use crate::result::Result;
     use crate::test_tx::test_tx;
@@ -111,6 +111,10 @@ mod tests {
         let (genesis_block, _mint_keypair) = GenesisBlock::new(2);
         let bank = Arc::new(Bank::new(&genesis_block));
         let prev_hash = bank.last_blockhash();
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blocktree =
+                Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger");
             let (poh_recorder, entry_receiver) = PohRecorder::new(
                 bank.tick_height(),
                 prev_hash,
@@ -118,6 +122,7 @@ mod tests {
                 Some(4),
                 bank.ticks_per_slot(),
                 &Pubkey::default(),
+                &Arc::new(blocktree),
             );
             let poh_recorder = Arc::new(Mutex::new(poh_recorder));
             let exit = Arc::new(AtomicBool::new(false));
@@ -194,4 +199,6 @@ mod tests {
             let _ = poh_service.join().unwrap();
             let _ = entry_producer.join().unwrap();
         }
+        Blocktree::destroy(&ledger_path).unwrap();
+    }
 }

core/src/replay_stage.rs

@@ -147,7 +147,13 @@ impl ReplayStage {
                             &cluster_info,
                         );
 
-                        Self::reset_poh_recorder(&my_id, &bank, &poh_recorder, ticks_per_slot);
+                        Self::reset_poh_recorder(
+                            &my_id,
+                            &blocktree,
+                            &bank,
+                            &poh_recorder,
+                            ticks_per_slot,
+                        );
 
                         is_tpu_bank_active = false;
                     }
@@ -171,7 +177,6 @@ impl ReplayStage {
                             &bank_forks,
                             &poh_recorder,
                             &cluster_info,
-                            &blocktree,
                             poh_slot,
                             reached_leader_tick,
                             grace_ticks,
@@ -200,35 +205,11 @@ impl ReplayStage {
         bank_forks: &Arc<RwLock<BankForks>>,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
         cluster_info: &Arc<RwLock<ClusterInfo>>,
-        blocktree: &Blocktree,
         poh_slot: u64,
         reached_leader_tick: bool,
         grace_ticks: u64,
     ) {
         trace!("{} checking poh slot {}", my_id, poh_slot);
-        if blocktree.meta(poh_slot).unwrap().is_some() {
-            // We've already broadcasted entries for this slot, skip it
-
-            // Since we are skipping our leader slot, let's tell poh recorder when we should be
-            // leader again
-            if reached_leader_tick {
-                let _ = bank_forks.read().unwrap().get(poh_slot).map(|bank| {
-                    let next_leader_slot =
-                        leader_schedule_utils::next_leader_slot(&my_id, bank.slot(), &bank);
-                    let mut poh = poh_recorder.lock().unwrap();
-                    let start_slot = poh.start_slot();
-                    poh.reset(
-                        bank.tick_height(),
-                        bank.last_blockhash(),
-                        start_slot,
-                        next_leader_slot,
-                        bank.ticks_per_slot(),
-                    );
-                });
-            }
-
-            return;
-        }
         if bank_forks.read().unwrap().get(poh_slot).is_none() {
             let parent_slot = poh_recorder.lock().unwrap().start_slot();
             let parent = {
@@ -332,11 +313,13 @@ impl ReplayStage {
 
     fn reset_poh_recorder(
         my_id: &Pubkey,
+        blocktree: &Blocktree,
         bank: &Arc<Bank>,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
         ticks_per_slot: u64,
     ) {
-        let next_leader_slot = leader_schedule_utils::next_leader_slot(&my_id, bank.slot(), &bank);
+        let next_leader_slot =
+            leader_schedule_utils::next_leader_slot(&my_id, bank.slot(), &bank, Some(blocktree));
         poh_recorder.lock().unwrap().reset(
             bank.tick_height(),
             bank.last_blockhash(),
@@ -635,7 +618,8 @@ mod test {
             let bank = bank_forks.working_bank();
 
             let blocktree = Arc::new(blocktree);
-            let (exit, poh_recorder, poh_service, _entry_receiver) = create_test_recorder(&bank);
+            let (exit, poh_recorder, poh_service, _entry_receiver) =
+                create_test_recorder(&bank, &blocktree);
             let (ledger_writer_sender, ledger_writer_receiver) = channel();
             let (replay_stage, _slot_full_receiver) = ReplayStage::new(
                 &my_keypair.pubkey(),

core/src/tvu.rs

@@ -208,8 +208,10 @@ pub mod tests {
         let blocktree_path = get_tmp_ledger_path!();
         let (blocktree, l_receiver) = Blocktree::open_with_signal(&blocktree_path)
             .expect("Expected to successfully open ledger");
+        let blocktree = Arc::new(blocktree);
         let bank = bank_forks.working_bank();
-        let (exit, poh_recorder, poh_service, _entry_receiver) = create_test_recorder(&bank);
+        let (exit, poh_recorder, poh_service, _entry_receiver) =
+            create_test_recorder(&bank, &blocktree);
         let voting_keypair = Keypair::new();
         let (storage_entry_sender, storage_entry_receiver) = channel();
         let tvu = Tvu::new(
@@ -225,7 +227,7 @@ pub mod tests {
                     fetch: target1.sockets.tvu,
                 }
             },
-            Arc::new(blocktree),
+            blocktree,
             STORAGE_ROTATE_TEST_COUNT,
             &StorageState::default(),
             None,

core/tests/tvu.rs

@@ -99,8 +99,10 @@ fn test_replay() {
     let dr_1 = new_gossip(cref1.clone(), target1.sockets.gossip, &exit);
 
     let voting_keypair = Keypair::new();
+    let blocktree = Arc::new(blocktree);
+    {
         let (poh_service_exit, poh_recorder, poh_service, _entry_receiver) =
-        create_test_recorder(&bank);
+            create_test_recorder(&bank, &blocktree);
         let (storage_sender, storage_receiver) = channel();
         let tvu = Tvu::new(
             &voting_keypair.pubkey(),
@@ -115,7 +117,7 @@ fn test_replay() {
                     fetch: target1.sockets.tvu,
                 }
             },
-        Arc::new(blocktree),
+            blocktree,
             STORAGE_ROTATE_TEST_COUNT,
             &StorageState::default(),
             None,
@@ -188,6 +190,7 @@ fn test_replay() {
         dr_1.join().unwrap();
         t_receiver.join().unwrap();
         t_responder.join().unwrap();
+    }
     Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
     let _ignored = remove_dir_all(&blocktree_path);
 }