Don't stop to find newer cluster-confirmed roots (bp #14557) (#14559)

* Don't stop to find newer cluster-confirmed roots (#14557)

* Don't stop to find newer cluster-confirmed roots

* Fix and add new tests

* nits

(cherry picked from commit e95ebcf864)

# Conflicts:
#	ledger/src/blockstore_processor.rs

* My blind merge conflict edit!

* blind 2

Co-authored-by: Ryo Onodera <ryoqun@gmail.com>

ledger/src/blockstore_processor.rs

@@ -804,7 +804,7 @@ fn load_frozen_forks(
     let mut last_status_report = Instant::now();
     let mut last_free = Instant::now();
     let mut pending_slots = vec![];
-    let mut last_root_slot = root_bank.slot();
+    let mut last_root = root_bank.slot();
     let mut slots_elapsed = 0;
     let mut txs = 0;
     let blockstore_max_root = blockstore.max_root();
@@ -832,7 +832,7 @@ fn load_frozen_forks(
             info!(
                 "processing ledger: slot={}, last root slot={} slots={} slots/s={:?} txs/s={}",
                 slot,
-                last_root_slot,
+                last_root,
                 slots_elapsed,
                 slots_elapsed as f32 / secs,
                 txs as f32 / secs,
@@ -869,7 +869,7 @@ fn load_frozen_forks(
         // If we've reached the last known root in blockstore, start looking
         // for newer cluster confirmed roots
         let new_root_bank = {
-            if *root == max_root {
+            if *root >= max_root {
                 supermajority_root_from_vote_accounts(
                     bank.slot(),
                     bank.total_epoch_stake(),
@@ -899,7 +899,8 @@ fn load_frozen_forks(
 
         if let Some(new_root_bank) = new_root_bank {
             *root = new_root_bank.slot();
-            last_root_slot = new_root_bank.slot();
+            last_root = new_root_bank.slot();
+
             leader_schedule_cache.set_root(&new_root_bank);
             new_root_bank.squash();
 
@@ -919,7 +920,7 @@ fn load_frozen_forks(
 
         trace!(
             "Bank for {}slot {} is complete. {} bytes allocated",
-            if last_root_slot == slot { "root " } else { "" },
+            if last_root == slot { "root " } else { "" },
             slot,
             allocated.since(initial_allocation)
         );
@@ -3051,6 +3052,8 @@ pub mod tests {
                   ...    minor fork
                   /
             `last_slot`
+                 |
+            `really_last_slot`
         */
         let starting_fork_slot = 5;
         let mut main_fork = tr(starting_fork_slot);
@@ -3058,10 +3061,12 @@ pub mod tests {
 
         // Make enough slots to make a root slot > blockstore_root
         let expected_root_slot = starting_fork_slot + blockstore_root.unwrap_or(0);
+        let really_expected_root_slot = expected_root_slot + 1;
         let last_main_fork_slot = expected_root_slot + MAX_LOCKOUT_HISTORY as u64 + 1;
+        let really_last_main_fork_slot = last_main_fork_slot + 1;
 
         // Make `minor_fork`
-        let last_minor_fork_slot = last_main_fork_slot + 1;
+        let last_minor_fork_slot = really_last_main_fork_slot + 1;
         let minor_fork = tr(last_minor_fork_slot);
 
         // Make 'main_fork`
@@ -3096,6 +3101,7 @@ pub mod tests {
         let (bank_forks, _leader_schedule) =
             process_blockstore(&genesis_config, &blockstore, Vec::new(), opts.clone()).unwrap();
 
+        // prepare to add votes
         let last_vote_bank_hash = bank_forks.get(last_main_fork_slot - 1).unwrap().hash();
         let last_vote_blockhash = bank_forks
             .get(last_main_fork_slot - 1)
@@ -3125,12 +3131,12 @@ pub mod tests {
         );
 
         let (bank_forks, _leader_schedule) =
-            process_blockstore(&genesis_config, &blockstore, Vec::new(), opts).unwrap();
+            process_blockstore(&genesis_config, &blockstore, Vec::new(), opts.clone()).unwrap();
 
         assert_eq!(bank_forks.root(), expected_root_slot);
         assert_eq!(
             bank_forks.frozen_banks().len() as u64,
-            last_minor_fork_slot - expected_root_slot + 1
+            last_minor_fork_slot - really_expected_root_slot + 1
         );
 
         // Minor fork at `last_main_fork_slot + 1` was above the `expected_root_slot`
@@ -3138,6 +3144,10 @@ pub mod tests {
         //
         // Fork at slot 2 was purged because it was below the `expected_root_slot`
         for slot in 0..=last_minor_fork_slot {
+            // this slot will be created below
+            if slot == really_last_main_fork_slot {
+                continue;
+            }
             if slot >= expected_root_slot {
                 let bank = bank_forks.get(slot).unwrap();
                 assert_eq!(bank.slot(), slot);
@@ -3146,11 +3156,48 @@ pub mod tests {
                 assert!(bank_forks.get(slot).is_none());
             }
         }
+
+        // really prepare to add votes
+        let last_vote_bank_hash = bank_forks.get(last_main_fork_slot).unwrap().hash();
+        let last_vote_blockhash = bank_forks
+            .get(last_main_fork_slot)
+            .unwrap()
+            .last_blockhash();
+        let slots: Vec<_> = vec![last_main_fork_slot];
+        let vote_tx = vote_transaction::new_vote_transaction(
+            slots,
+            last_vote_bank_hash,
+            last_vote_blockhash,
+            &leader_keypair,
+            &validator_keypairs.vote_keypair,
+            &validator_keypairs.vote_keypair,
+            None,
+        );
+
+        // Add votes to `really_last_slot` so that `root` will be confirmed again
+        make_slot_with_vote_tx(
+            &blockstore,
+            ticks_per_slot,
+            really_last_main_fork_slot,
+            last_main_fork_slot,
+            &last_vote_blockhash,
+            vote_tx,
+            &leader_keypair,
+        );
+
+        let (bank_forks, _leader_schedule) =
+            process_blockstore(&genesis_config, &blockstore, Vec::new(), opts).unwrap();
+
+        assert_eq!(bank_forks.root(), really_expected_root_slot);
     }
 
     #[test]
-    fn test_process_blockstore_with_supermajority_root() {
+    fn test_process_blockstore_with_supermajority_root_without_blockstore_root() {
         run_test_process_blockstore_with_supermajority_root(None);
+    }
+
+    #[test]
+    fn test_process_blockstore_with_supermajority_root_with_blockstore_root() {
         run_test_process_blockstore_with_supermajority_root(Some(1))
     }