prioritize slot repairs for unknown last index and close to completion (#21070)

core/src/lib.rs

@@ -34,6 +34,7 @@ pub mod outstanding_requests;
 pub mod packet_hasher;
 pub mod progress_map;
 pub mod qos_service;
+pub mod repair_generic_traversal;
 pub mod repair_response;
 pub mod repair_service;
 pub mod repair_weight;

core/src/repair_generic_traversal.rs

@@ -0,0 +1,312 @@
+use crate::{
+    heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice, repair_service::RepairService,
+    serve_repair::ShredRepairType, tree_diff::TreeDiff,
+};
+use solana_ledger::{blockstore::Blockstore, blockstore_meta::SlotMeta};
+use solana_sdk::{clock::Slot, hash::Hash};
+use std::collections::{HashMap, HashSet};
+
+struct GenericTraversal<'a> {
+    tree: &'a HeaviestSubtreeForkChoice,
+    pending: Vec<Slot>,
+}
+
+impl<'a> GenericTraversal<'a> {
+    pub fn new(tree: &'a HeaviestSubtreeForkChoice) -> Self {
+        Self {
+            tree,
+            pending: vec![tree.root().0],
+        }
+    }
+}
+
+impl<'a> Iterator for GenericTraversal<'a> {
+    type Item = Slot;
+    fn next(&mut self) -> Option<Self::Item> {
+        let next = self.pending.pop();
+        if let Some(slot) = next {
+            let children: Vec<_> = self
+                .tree
+                .children(&(slot, Hash::default()))
+                .unwrap()
+                .iter()
+                .map(|(child_slot, _)| *child_slot)
+                .collect();
+            self.pending.extend(children);
+        }
+        next
+    }
+}
+
+pub fn get_unknown_last_index(
+    tree: &HeaviestSubtreeForkChoice,
+    blockstore: &Blockstore,
+    slot_meta_cache: &mut HashMap<Slot, Option<SlotMeta>>,
+    processed_slots: &mut HashSet<Slot>,
+    limit: usize,
+) -> Vec<ShredRepairType> {
+    let iter = GenericTraversal::new(tree);
+    let mut unknown_last = Vec::new();
+    for slot in iter {
+        if processed_slots.contains(&slot) {
+            continue;
+        }
+        let slot_meta = slot_meta_cache
+            .entry(slot)
+            .or_insert_with(|| blockstore.meta(slot).unwrap());
+        if let Some(slot_meta) = slot_meta {
+            if slot_meta.known_last_index().is_none() {
+                let shred_index = blockstore.get_index(slot).unwrap();
+                let num_processed_shreds = if let Some(shred_index) = shred_index {
+                    shred_index.data().num_shreds() as u64
+                } else {
+                    slot_meta.consumed
+                };
+                unknown_last.push((slot, slot_meta.received, num_processed_shreds));
+                processed_slots.insert(slot);
+            }
+        }
+    }
+    // prioritize slots with more received shreds
+    unknown_last.sort_by(|(_, _, count1), (_, _, count2)| count2.cmp(count1));
+    unknown_last
+        .iter()
+        .take(limit)
+        .map(|(slot, received, _)| ShredRepairType::HighestShred(*slot, *received))
+        .collect()
+}
+
+fn get_unrepaired_path(
+    start_slot: Slot,
+    blockstore: &Blockstore,
+    slot_meta_cache: &mut HashMap<Slot, Option<SlotMeta>>,
+    visited: &mut HashSet<Slot>,
+) -> Vec<Slot> {
+    let mut path = Vec::new();
+    let mut slot = start_slot;
+    while !visited.contains(&slot) {
+        visited.insert(slot);
+        let slot_meta = slot_meta_cache
+            .entry(slot)
+            .or_insert_with(|| blockstore.meta(slot).unwrap());
+        if let Some(slot_meta) = slot_meta {
+            if slot_meta.is_full() {
+                break;
+            }
+            path.push(slot);
+            slot = slot_meta.parent_slot;
+        }
+    }
+    path.reverse();
+    path
+}
+
+pub fn get_closest_completion(
+    tree: &HeaviestSubtreeForkChoice,
+    blockstore: &Blockstore,
+    slot_meta_cache: &mut HashMap<Slot, Option<SlotMeta>>,
+    processed_slots: &mut HashSet<Slot>,
+    limit: usize,
+) -> Vec<ShredRepairType> {
+    let mut v: Vec<(Slot, u64)> = Vec::default();
+    let iter = GenericTraversal::new(tree);
+    for slot in iter {
+        if processed_slots.contains(&slot) {
+            continue;
+        }
+        let slot_meta = slot_meta_cache
+            .entry(slot)
+            .or_insert_with(|| blockstore.meta(slot).unwrap());
+        if let Some(slot_meta) = slot_meta {
+            if slot_meta.is_full() {
+                continue;
+            }
+            if let Some(last_index) = slot_meta.known_last_index() {
+                let shred_index = blockstore.get_index(slot).unwrap();
+                let dist = if let Some(shred_index) = shred_index {
+                    let shred_count = shred_index.data().num_shreds() as u64;
+                    last_index - shred_count
+                } else {
+                    last_index - slot_meta.consumed
+                };
+                v.push((slot, dist));
+                processed_slots.insert(slot);
+            }
+        }
+    }
+    v.sort_by(|(_, d1), (_, d2)| d1.cmp(d2));
+
+    let mut visited = HashSet::new();
+    let mut repairs = Vec::new();
+    for (slot, _) in v {
+        if repairs.len() >= limit {
+            break;
+        }
+        // attempt to repair heaviest slots starting with their parents
+        let path = get_unrepaired_path(slot, blockstore, slot_meta_cache, &mut visited);
+        for slot in path {
+            if repairs.len() >= limit {
+                break;
+            }
+            let slot_meta = slot_meta_cache.get(&slot).unwrap().as_ref().unwrap();
+            let new_repairs = RepairService::generate_repairs_for_slot(
+                blockstore,
+                slot,
+                slot_meta,
+                limit - repairs.len(),
+            );
+            repairs.extend(new_repairs);
+        }
+    }
+
+    repairs
+}
+
+#[cfg(test)]
+pub mod test {
+    use super::*;
+    use solana_ledger::{
+        blockstore::{Blockstore, MAX_TURBINE_PROPAGATION_IN_MS},
+        get_tmp_ledger_path,
+    };
+    use solana_sdk::hash::Hash;
+    use std::{thread::sleep, time::Duration};
+    use trees::{tr, Tree, TreeWalk};
+
+    #[test]
+    fn test_get_unknown_last_index() {
+        let (blockstore, heaviest_subtree_fork_choice) = setup_forks();
+        let last_shred = blockstore.meta(0).unwrap().unwrap().received;
+        let mut slot_meta_cache = HashMap::default();
+        let mut processed_slots = HashSet::default();
+        let repairs = get_unknown_last_index(
+            &heaviest_subtree_fork_choice,
+            &blockstore,
+            &mut slot_meta_cache,
+            &mut processed_slots,
+            10,
+        );
+        assert_eq!(
+            repairs,
+            [0, 1, 3, 5, 2, 4]
+                .iter()
+                .map(|slot| ShredRepairType::HighestShred(*slot, last_shred))
+                .collect::<Vec<_>>()
+        );
+    }
+
+    #[test]
+    fn test_get_closest_completion() {
+        let (blockstore, heaviest_subtree_fork_choice) = setup_forks();
+        let mut slot_meta_cache = HashMap::default();
+        let mut processed_slots = HashSet::default();
+        let repairs = get_closest_completion(
+            &heaviest_subtree_fork_choice,
+            &blockstore,
+            &mut slot_meta_cache,
+            &mut processed_slots,
+            10,
+        );
+        assert_eq!(repairs, []);
+
+        let forks = tr(0) / (tr(1) / (tr(2) / (tr(4))) / (tr(3) / (tr(5))));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Blockstore::open(&ledger_path).unwrap();
+        add_tree_with_missing_shreds(
+            &blockstore,
+            forks.clone(),
+            false,
+            true,
+            100,
+            Hash::default(),
+        );
+        let heaviest_subtree_fork_choice = HeaviestSubtreeForkChoice::new_from_tree(forks);
+        sleep(Duration::from_millis(MAX_TURBINE_PROPAGATION_IN_MS));
+        let mut slot_meta_cache = HashMap::default();
+        let mut processed_slots = HashSet::default();
+        let repairs = get_closest_completion(
+            &heaviest_subtree_fork_choice,
+            &blockstore,
+            &mut slot_meta_cache,
+            &mut processed_slots,
+            2,
+        );
+        assert_eq!(
+            repairs,
+            [ShredRepairType::Shred(0, 3), ShredRepairType::Shred(1, 3)]
+        );
+    }
+
+    fn add_tree_with_missing_shreds(
+        blockstore: &Blockstore,
+        forks: Tree<Slot>,
+        is_orphan: bool,
+        is_slot_complete: bool,
+        num_ticks: u64,
+        starting_hash: Hash,
+    ) {
+        let mut walk = TreeWalk::from(forks);
+        let mut blockhashes = HashMap::new();
+        while let Some(visit) = walk.get() {
+            let slot = *visit.node().data();
+            if blockstore.meta(slot).unwrap().is_some()
+                && blockstore.orphan(slot).unwrap().is_none()
+            {
+                // If slot exists in blockstore and is not an orphan, then skip it
+                walk.forward();
+                continue;
+            }
+            let parent = walk.get_parent().map(|n| *n.data());
+            if parent.is_some() || !is_orphan {
+                let parent_hash = parent
+                    // parent won't exist for first node in a tree where
+                    // `is_orphan == true`
+                    .and_then(|parent| blockhashes.get(&parent))
+                    .unwrap_or(&starting_hash);
+                let entries = solana_entry::entry::create_ticks(
+                    num_ticks * (std::cmp::max(1, slot - parent.unwrap_or(slot))),
+                    0,
+                    *parent_hash,
+                );
+                blockhashes.insert(slot, entries.last().unwrap().hash);
+
+                let mut shreds = solana_ledger::blockstore::entries_to_test_shreds(
+                    entries.clone(),
+                    slot,
+                    parent.unwrap_or(slot),
+                    is_slot_complete,
+                    0,
+                );
+
+                // remove next to last shred
+                let shred = shreds.pop().unwrap();
+                shreds.pop().unwrap();
+                shreds.push(shred);
+
+                blockstore.insert_shreds(shreds, None, false).unwrap();
+            }
+            walk.forward();
+        }
+    }
+
+    fn setup_forks() -> (Blockstore, HeaviestSubtreeForkChoice) {
+        /*
+            Build fork structure:
+                 slot 0
+                   |
+                 slot 1
+                 /    \
+            slot 2    |
+               |    slot 3
+            slot 4    |
+                    slot 5
+        */
+
+        let forks = tr(0) / (tr(1) / (tr(2) / (tr(4))) / (tr(3) / (tr(5))));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Blockstore::open(&ledger_path).unwrap();
+        blockstore.add_tree(forks.clone(), false, false, 2, Hash::default());
+
+        (blockstore, HeaviestSubtreeForkChoice::new_from_tree(forks))
+    }
+}

core/src/repair_service.rs

@@ -95,6 +95,8 @@ pub struct RepairTiming {
     pub add_votes_elapsed: u64,
     pub get_best_orphans_elapsed: u64,
     pub get_best_shreds_elapsed: u64,
+    pub get_unknown_last_index_elapsed: u64,
+    pub get_closest_completion_elapsed: u64,
     pub send_repairs_elapsed: u64,
     pub build_repairs_batch_elapsed: u64,
     pub batch_send_repairs_elapsed: u64,
@@ -118,11 +120,50 @@ impl RepairTiming {
     }
 }
 
+#[derive(Default, Debug)]
+pub struct BestRepairsStats {
+    pub call_count: u64,
+    pub num_orphan_slots: u64,
+    pub num_orphan_repairs: u64,
+    pub num_best_shreds_slots: u64,
+    pub num_best_shreds_repairs: u64,
+    pub num_unknown_last_index_slots: u64,
+    pub num_unknown_last_index_repairs: u64,
+    pub num_closest_completion_slots: u64,
+    pub num_closest_completion_repairs: u64,
+}
+
+impl BestRepairsStats {
+    pub fn update(
+        &mut self,
+        num_orphan_slots: u64,
+        num_orphan_repairs: u64,
+        num_best_shreds_slots: u64,
+        num_best_shreds_repairs: u64,
+        num_unknown_last_index_slots: u64,
+        num_unknown_last_index_repairs: u64,
+        num_closest_completion_slots: u64,
+        num_closest_completion_repairs: u64,
+    ) {
+        self.call_count += 1;
+        self.num_orphan_slots += num_orphan_slots;
+        self.num_orphan_repairs += num_orphan_repairs;
+        self.num_best_shreds_slots += num_best_shreds_slots;
+        self.num_best_shreds_repairs += num_best_shreds_repairs;
+        self.num_unknown_last_index_slots += num_unknown_last_index_slots;
+        self.num_unknown_last_index_repairs += num_unknown_last_index_repairs;
+        self.num_closest_completion_slots += num_closest_completion_slots;
+        self.num_closest_completion_repairs += num_closest_completion_repairs;
+    }
+}
+
 pub const MAX_REPAIR_LENGTH: usize = 512;
 pub const MAX_REPAIR_PER_DUPLICATE: usize = 20;
 pub const MAX_DUPLICATE_WAIT_MS: usize = 10_000;
 pub const REPAIR_MS: u64 = 100;
 pub const MAX_ORPHANS: usize = 5;
+pub const MAX_UNKNOWN_LAST_INDEX_REPAIRS: usize = 10;
+pub const MAX_CLOSEST_COMPLETION_REPAIRS: usize = 100;
 
 #[derive(Clone)]
 pub struct RepairInfo {
@@ -210,6 +251,7 @@ impl RepairService {
         let id = repair_info.cluster_info.id();
         let mut repair_stats = RepairStats::default();
         let mut repair_timing = RepairTiming::default();
+        let mut best_repairs_stats = BestRepairsStats::default();
         let mut last_stats = Instant::now();
         let duplicate_slot_repair_statuses: HashMap<Slot, DuplicateSlotRepairStatus> =
             HashMap::new();
@@ -257,15 +299,20 @@ impl RepairService {
                 );
                 add_votes_elapsed.stop();
 
-                repair_weight.get_best_weighted_repairs(
+                let repairs = repair_weight.get_best_weighted_repairs(
                     blockstore,
                     root_bank.epoch_stakes_map(),
                     root_bank.epoch_schedule(),
                     MAX_ORPHANS,
                     MAX_REPAIR_LENGTH,
+                    MAX_UNKNOWN_LAST_INDEX_REPAIRS,
+                    MAX_CLOSEST_COMPLETION_REPAIRS,
                     &duplicate_slot_repair_statuses,
                     Some(&mut repair_timing),
-                )
+                    Some(&mut best_repairs_stats),
+                );
+
+                repairs
             };
 
             let mut build_repairs_batch_elapsed = Measure::start("build_repairs_batch_elapsed");
@@ -362,6 +409,16 @@ impl RepairService {
                         repair_timing.get_best_shreds_elapsed,
                         i64
                     ),
+                    (
+                        "get-unknown-last-index-elapsed",
+                        repair_timing.get_unknown_last_index_elapsed,
+                        i64
+                    ),
+                    (
+                        "get-closest-completion-elapsed",
+                        repair_timing.get_closest_completion_elapsed,
+                        i64
+                    ),
                     (
                         "send-repairs-elapsed",
                         repair_timing.send_repairs_elapsed,
@@ -378,8 +435,45 @@ impl RepairService {
                         i64
                     ),
                 );
+                datapoint_info!(
+                    "serve_repair-best-repairs",
+                    ("call-count", best_repairs_stats.call_count, i64),
+                    ("orphan-slots", best_repairs_stats.num_orphan_slots, i64),
+                    ("orphan-repairs", best_repairs_stats.num_orphan_repairs, i64),
+                    (
+                        "best-shreds-slots",
+                        best_repairs_stats.num_best_shreds_slots,
+                        i64
+                    ),
+                    (
+                        "best-shreds-repairs",
+                        best_repairs_stats.num_best_shreds_repairs,
+                        i64
+                    ),
+                    (
+                        "unknown-last-index-slots",
+                        best_repairs_stats.num_unknown_last_index_slots,
+                        i64
+                    ),
+                    (
+                        "unknown-last-index-repairs",
+                        best_repairs_stats.num_unknown_last_index_repairs,
+                        i64
+                    ),
+                    (
+                        "closest-completion-slots",
+                        best_repairs_stats.num_closest_completion_slots,
+                        i64
+                    ),
+                    (
+                        "closest-completion-repairs",
+                        best_repairs_stats.num_closest_completion_repairs,
+                        i64
+                    ),
+                );
                 repair_stats = RepairStats::default();
                 repair_timing = RepairTiming::default();
+                best_repairs_stats = BestRepairsStats::default();
                 last_stats = Instant::now();
             }
             sleep(Duration::from_millis(REPAIR_MS));
@@ -474,7 +568,7 @@ impl RepairService {
         }
     }
 
-    #[allow(dead_code)]
+    #[cfg_attr(not(test), allow(dead_code))]
     fn generate_duplicate_repairs_for_slot(
         blockstore: &Blockstore,
         slot: Slot,
@@ -499,7 +593,7 @@ impl RepairService {
         }
     }
 
-    #[allow(dead_code)]
+    #[cfg_attr(not(test), allow(dead_code))]
     fn generate_and_send_duplicate_repairs(
         duplicate_slot_repair_statuses: &mut HashMap<Slot, DuplicateSlotRepairStatus>,
         cluster_slots: &ClusterSlots,
@@ -550,7 +644,7 @@ impl RepairService {
         })
     }
 
-    #[allow(dead_code)]
+    #[cfg_attr(not(test), allow(dead_code))]
     fn serialize_and_send_request(
         repair_type: &ShredRepairType,
         repair_socket: &UdpSocket,
@@ -566,7 +660,7 @@ impl RepairService {
         Ok(())
     }
 
-    #[allow(dead_code)]
+    #[cfg_attr(not(test), allow(dead_code))]
     fn update_duplicate_slot_repair_addr(
         slot: Slot,
         status: &mut DuplicateSlotRepairStatus,
@@ -659,8 +753,11 @@ mod test {
                     &EpochSchedule::default(),
                     MAX_ORPHANS,
                     MAX_REPAIR_LENGTH,
+                    MAX_UNKNOWN_LAST_INDEX_REPAIRS,
+                    MAX_CLOSEST_COMPLETION_REPAIRS,
                     &HashSet::default(),
                     None,
+                    None,
                 ),
                 vec![
                     ShredRepairType::Orphan(2),
@@ -693,8 +790,11 @@ mod test {
                     &EpochSchedule::default(),
                     MAX_ORPHANS,
                     MAX_REPAIR_LENGTH,
+                    MAX_UNKNOWN_LAST_INDEX_REPAIRS,
+                    MAX_CLOSEST_COMPLETION_REPAIRS,
                     &HashSet::default(),
-                    None
+                    None,
+                    None,
                 ),
                 vec![ShredRepairType::HighestShred(0, 0)]
             );
@@ -748,8 +848,11 @@ mod test {
                     &EpochSchedule::default(),
                     MAX_ORPHANS,
                     MAX_REPAIR_LENGTH,
+                    MAX_UNKNOWN_LAST_INDEX_REPAIRS,
+                    MAX_CLOSEST_COMPLETION_REPAIRS,
                     &HashSet::default(),
-                    None
+                    None,
+                    None,
                 ),
                 expected
             );
@@ -761,8 +864,11 @@ mod test {
                     &EpochSchedule::default(),
                     MAX_ORPHANS,
                     expected.len() - 2,
+                    MAX_UNKNOWN_LAST_INDEX_REPAIRS,
+                    MAX_CLOSEST_COMPLETION_REPAIRS,
                     &HashSet::default(),
-                    None
+                    None,
+                    None,
                 )[..],
                 expected[0..expected.len() - 2]
             );
@@ -799,8 +905,11 @@ mod test {
                     &EpochSchedule::default(),
                     MAX_ORPHANS,
                     MAX_REPAIR_LENGTH,
+                    MAX_UNKNOWN_LAST_INDEX_REPAIRS,
+                    MAX_CLOSEST_COMPLETION_REPAIRS,
                     &HashSet::default(),
-                    None
+                    None,
+                    None,
                 ),
                 expected
             );

core/src/repair_weight.rs

@@ -1,8 +1,14 @@
 use crate::{
-    heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice, repair_service::RepairTiming,
+    heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice,
-    repair_weighted_traversal, serve_repair::ShredRepairType, tree_diff::TreeDiff,
+    repair_generic_traversal::{get_closest_completion, get_unknown_last_index},
+    repair_service::{BestRepairsStats, RepairTiming},
+    repair_weighted_traversal,
+    serve_repair::ShredRepairType,
+    tree_diff::TreeDiff,
+};
+use solana_ledger::{
+    ancestor_iterator::AncestorIterator, blockstore::Blockstore, blockstore_meta::SlotMeta,
 };
-use solana_ledger::{ancestor_iterator::AncestorIterator, blockstore::Blockstore};
 use solana_measure::measure::Measure;
 use solana_runtime::{contains::Contains, epoch_stakes::EpochStakes};
 use solana_sdk::{
@@ -138,6 +144,7 @@ impl RepairWeight {
         }
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub fn get_best_weighted_repairs<'a>(
         &mut self,
         blockstore: &Blockstore,
@@ -145,29 +152,91 @@ impl RepairWeight {
         epoch_schedule: &EpochSchedule,
         max_new_orphans: usize,
         max_new_shreds: usize,
+        max_unknown_last_index_repairs: usize,
+        max_closest_completion_repairs: usize,
         ignore_slots: &impl Contains<'a, Slot>,
         repair_timing: Option<&mut RepairTiming>,
+        stats: Option<&mut BestRepairsStats>,
     ) -> Vec<ShredRepairType> {
         let mut repairs = vec![];
+        let mut processed_slots: HashSet<Slot> = vec![self.root].into_iter().collect();
+        let mut slot_meta_cache = HashMap::default();
+
         let mut get_best_orphans_elapsed = Measure::start("get_best_orphans");
         // Update the orphans in order from heaviest to least heavy
         self.get_best_orphans(
             blockstore,
+            &mut processed_slots,
             &mut repairs,
             epoch_stakes,
             epoch_schedule,
             max_new_orphans,
         );
+        let num_orphan_slots = processed_slots.len() - 1;
+        let num_orphan_repairs = repairs.len();
         get_best_orphans_elapsed.stop();
 
         let mut get_best_shreds_elapsed = Measure::start("get_best_shreds");
+        let mut best_shreds_repairs = Vec::default();
         // Find the best incomplete slots in rooted subtree
-        self.get_best_shreds(blockstore, &mut repairs, max_new_shreds, ignore_slots);
+        self.get_best_shreds(
+            blockstore,
+            &mut slot_meta_cache,
+            &mut best_shreds_repairs,
+            max_new_shreds,
+            ignore_slots,
+        );
+        let num_best_shreds_repairs = best_shreds_repairs.len();
+        let repair_slots_set: HashSet<Slot> =
+            best_shreds_repairs.iter().map(|r| r.slot()).collect();
+        let num_best_shreds_slots = repair_slots_set.len();
+        processed_slots.extend(repair_slots_set);
+        repairs.extend(best_shreds_repairs);
         get_best_shreds_elapsed.stop();
 
+        let mut get_unknown_last_index_elapsed = Measure::start("get_unknown_last_index");
+        let pre_num_slots = processed_slots.len();
+        let unknown_last_index_repairs = self.get_best_unknown_last_index(
+            blockstore,
+            &mut slot_meta_cache,
+            &mut processed_slots,
+            max_unknown_last_index_repairs,
+        );
+        let num_unknown_last_index_repairs = unknown_last_index_repairs.len();
+        let num_unknown_last_index_slots = processed_slots.len() - pre_num_slots;
+        repairs.extend(unknown_last_index_repairs);
+        get_unknown_last_index_elapsed.stop();
+
+        let mut get_closest_completion_elapsed = Measure::start("get_closest_completion");
+        let pre_num_slots = processed_slots.len();
+        let closest_completion_repairs = self.get_best_closest_completion(
+            blockstore,
+            &mut slot_meta_cache,
+            &mut processed_slots,
+            max_closest_completion_repairs,
+        );
+        let num_closest_completion_repairs = closest_completion_repairs.len();
+        let num_closest_completion_slots = processed_slots.len() - pre_num_slots;
+        repairs.extend(closest_completion_repairs);
+        get_closest_completion_elapsed.stop();
+
+        if let Some(stats) = stats {
+            stats.update(
+                num_orphan_slots as u64,
+                num_orphan_repairs as u64,
+                num_best_shreds_slots as u64,
+                num_best_shreds_repairs as u64,
+                num_unknown_last_index_slots as u64,
+                num_unknown_last_index_repairs as u64,
+                num_closest_completion_slots as u64,
+                num_closest_completion_repairs as u64,
+            );
+        }
         if let Some(repair_timing) = repair_timing {
             repair_timing.get_best_orphans_elapsed += get_best_orphans_elapsed.as_us();
             repair_timing.get_best_shreds_elapsed += get_best_shreds_elapsed.as_us();
+            repair_timing.get_unknown_last_index_elapsed += get_unknown_last_index_elapsed.as_us();
+            repair_timing.get_closest_completion_elapsed += get_closest_completion_elapsed.as_us();
         }
         repairs
     }
@@ -248,6 +317,7 @@ impl RepairWeight {
     fn get_best_shreds<'a>(
         &mut self,
         blockstore: &Blockstore,
+        slot_meta_cache: &mut HashMap<Slot, Option<SlotMeta>>,
         repairs: &mut Vec<ShredRepairType>,
         max_new_shreds: usize,
         ignore_slots: &impl Contains<'a, Slot>,
@@ -256,6 +326,7 @@ impl RepairWeight {
         repair_weighted_traversal::get_best_repair_shreds(
             root_tree,
             blockstore,
+            slot_meta_cache,
             repairs,
             max_new_shreds,
             ignore_slots,
@@ -265,6 +336,7 @@ impl RepairWeight {
     fn get_best_orphans(
         &mut self,
         blockstore: &Blockstore,
+        processed_slots: &mut HashSet<Slot>,
         repairs: &mut Vec<ShredRepairType>,
         epoch_stakes: &HashMap<Epoch, EpochStakes>,
         epoch_schedule: &EpochSchedule,
@@ -292,7 +364,7 @@ impl RepairWeight {
             if best_orphans.len() >= max_new_orphans {
                 break;
             }
-            if heaviest_tree_root == self.root {
+            if processed_slots.contains(&heaviest_tree_root) {
                 continue;
             }
             // Ignore trees that were merged in a previous iteration
@@ -307,6 +379,7 @@ impl RepairWeight {
                     if new_orphan_root != self.root && !best_orphans.contains(&new_orphan_root) {
                         best_orphans.insert(new_orphan_root);
                         repairs.push(ShredRepairType::Orphan(new_orphan_root));
+                        processed_slots.insert(new_orphan_root);
                     }
                 }
             }
@@ -319,6 +392,7 @@ impl RepairWeight {
                 if !best_orphans.contains(&new_orphan) {
                     repairs.push(ShredRepairType::Orphan(new_orphan));
                     best_orphans.insert(new_orphan);
+                    processed_slots.insert(new_orphan);
                 }
 
                 if best_orphans.len() == max_new_orphans {
@@ -328,6 +402,54 @@ impl RepairWeight {
         }
     }
 
+    fn get_best_unknown_last_index(
+        &mut self,
+        blockstore: &Blockstore,
+        slot_meta_cache: &mut HashMap<Slot, Option<SlotMeta>>,
+        processed_slots: &mut HashSet<Slot>,
+        max_new_repairs: usize,
+    ) -> Vec<ShredRepairType> {
+        let mut repairs = Vec::default();
+        for (_slot, tree) in self.trees.iter() {
+            if repairs.len() >= max_new_repairs {
+                break;
+            }
+            let new_repairs = get_unknown_last_index(
+                tree,
+                blockstore,
+                slot_meta_cache,
+                processed_slots,
+                max_new_repairs - repairs.len(),
+            );
+            repairs.extend(new_repairs);
+        }
+        repairs
+    }
+
+    fn get_best_closest_completion(
+        &mut self,
+        blockstore: &Blockstore,
+        slot_meta_cache: &mut HashMap<Slot, Option<SlotMeta>>,
+        processed_slots: &mut HashSet<Slot>,
+        max_new_repairs: usize,
+    ) -> Vec<ShredRepairType> {
+        let mut repairs = Vec::default();
+        for (_slot, tree) in self.trees.iter() {
+            if repairs.len() >= max_new_repairs {
+                break;
+            }
+            let new_repairs = get_closest_completion(
+                tree,
+                blockstore,
+                slot_meta_cache,
+                processed_slots,
+                max_new_repairs - repairs.len(),
+            );
+            repairs.extend(new_repairs);
+        }
+        repairs
+    }
+
     // Attempts to chain the orphan subtree rooted at `orphan_tree_root`
     // to any earlier subtree with new any ancestry information in `blockstore`.
     // Returns the earliest known ancestor of `heaviest_tree_root`.
@@ -852,8 +974,10 @@ mod test {
         // Ask for only 1 orphan. Because the orphans have the same weight,
         // should prioritize smaller orphan first
         let mut repairs = vec![];
+        let mut processed_slots: HashSet<Slot> = vec![repair_weight.root].into_iter().collect();
         repair_weight.get_best_orphans(
             &blockstore,
+            &mut processed_slots,
             &mut repairs,
             bank.epoch_stakes_map(),
             bank.epoch_schedule(),
@@ -879,6 +1003,7 @@ mod test {
         // New vote on same orphan branch, without any new slot chaining
         // information blockstore should not resolve the orphan
         repairs = vec![];
+        processed_slots = vec![repair_weight.root].into_iter().collect();
         let votes = vec![(10, vec![vote_pubkeys[0]])];
         repair_weight.add_votes(
             &blockstore,
@@ -888,6 +1013,7 @@ mod test {
         );
         repair_weight.get_best_orphans(
             &blockstore,
+            &mut processed_slots,
             &mut repairs,
             bank.epoch_stakes_map(),
             bank.epoch_schedule(),
@@ -898,8 +1024,10 @@ mod test {
 
         // Ask for 2 orphans, should return all the orphans
         repairs = vec![];
+        processed_slots = vec![repair_weight.root].into_iter().collect();
         repair_weight.get_best_orphans(
             &blockstore,
+            &mut processed_slots,
             &mut repairs,
             bank.epoch_stakes_map(),
             bank.epoch_schedule(),
@@ -911,6 +1039,7 @@ mod test {
 
         // If one orphan gets heavier, should pick that one
         repairs = vec![];
+        processed_slots = vec![repair_weight.root].into_iter().collect();
         let votes = vec![(20, vec![vote_pubkeys[0]])];
         repair_weight.add_votes(
             &blockstore,
@@ -920,6 +1049,7 @@ mod test {
         );
         repair_weight.get_best_orphans(
             &blockstore,
+            &mut processed_slots,
             &mut repairs,
             bank.epoch_stakes_map(),
             bank.epoch_schedule(),
@@ -931,10 +1061,12 @@ mod test {
         // Resolve the orphans, should now return no
         // orphans
         repairs = vec![];
+        processed_slots = vec![repair_weight.root].into_iter().collect();
         blockstore.add_tree(tr(6) / (tr(8)), true, true, 2, Hash::default());
         blockstore.add_tree(tr(11) / (tr(20)), true, true, 2, Hash::default());
         repair_weight.get_best_orphans(
             &blockstore,
+            &mut processed_slots,
             &mut repairs,
             bank.epoch_stakes_map(),
             bank.epoch_schedule(),
@@ -967,9 +1099,11 @@ mod test {
         // orphan in the `trees` map, we should search for
         // exactly one more of the remaining two
         let mut repairs = vec![];
+        let mut processed_slots: HashSet<Slot> = vec![repair_weight.root].into_iter().collect();
         blockstore.add_tree(tr(100) / (tr(101)), true, true, 2, Hash::default());
         repair_weight.get_best_orphans(
             &blockstore,
+            &mut processed_slots,
             &mut repairs,
             bank.epoch_stakes_map(),
             bank.epoch_schedule(),
@@ -981,8 +1115,10 @@ mod test {
 
         // If we ask for 3 orphans, we should get all of them
         let mut repairs = vec![];
+        processed_slots = vec![repair_weight.root].into_iter().collect();
         repair_weight.get_best_orphans(
             &blockstore,
+            &mut processed_slots,
             &mut repairs,
             bank.epoch_stakes_map(),
             bank.epoch_schedule(),

core/src/repair_weighted_traversal.rs

@@ -2,7 +2,7 @@ use crate::{
     heaviest_subtree_fork_choice::HeaviestSubtreeForkChoice, repair_service::RepairService,
     serve_repair::ShredRepairType, tree_diff::TreeDiff,
 };
-use solana_ledger::blockstore::Blockstore;
+use solana_ledger::{blockstore::Blockstore, blockstore_meta::SlotMeta};
 use solana_runtime::contains::Contains;
 use solana_sdk::{clock::Slot, hash::Hash};
 use std::collections::{HashMap, HashSet};
@@ -29,7 +29,7 @@ struct RepairWeightTraversal<'a> {
 }
 
 impl<'a> RepairWeightTraversal<'a> {
-    pub fn new(tree: &'a HeaviestSubtreeForkChoice) -> Self {
+    fn new(tree: &'a HeaviestSubtreeForkChoice) -> Self {
         Self {
             tree,
             pending: vec![Visit::Unvisited(tree.root().0)],
@@ -73,6 +73,7 @@ impl<'a> Iterator for RepairWeightTraversal<'a> {
 pub fn get_best_repair_shreds<'a>(
     tree: &HeaviestSubtreeForkChoice,
     blockstore: &Blockstore,
+    slot_meta_cache: &mut HashMap<Slot, Option<SlotMeta>>,
     repairs: &mut Vec<ShredRepairType>,
     max_new_shreds: usize,
     ignore_slots: &impl Contains<'a, Slot>,
@@ -81,7 +82,6 @@ pub fn get_best_repair_shreds<'a>(
     let max_repairs = initial_len + max_new_shreds;
     let weighted_iter = RepairWeightTraversal::new(tree);
     let mut visited_set = HashSet::new();
-    let mut slot_meta_cache = HashMap::new();
     for next in weighted_iter {
         if repairs.len() > max_repairs {
             break;
@@ -215,10 +215,12 @@ pub mod test {
         // `blockstore` and `heaviest_subtree_fork_choice` match exactly, so should
         // return repairs for all slots (none are completed) in order of traversal
         let mut repairs = vec![];
+        let mut slot_meta_cache = HashMap::default();
         let last_shred = blockstore.meta(0).unwrap().unwrap().received;
         get_best_repair_shreds(
             &heaviest_subtree_fork_choice,
             &blockstore,
+            &mut slot_meta_cache,
             &mut repairs,
             6,
             &HashSet::default(),
@@ -234,6 +236,7 @@ pub mod test {
         // Add some leaves to blockstore, attached to the current best leaf, should prioritize
         // repairing those new leaves before trying other branches
         repairs = vec![];
+        slot_meta_cache = HashMap::default();
         let best_overall_slot = heaviest_subtree_fork_choice.best_overall_slot().0;
         assert_eq!(best_overall_slot, 4);
         blockstore.add_tree(
@@ -246,6 +249,7 @@ pub mod test {
         get_best_repair_shreds(
             &heaviest_subtree_fork_choice,
             &blockstore,
+            &mut slot_meta_cache,
             &mut repairs,
             6,
             &HashSet::default(),
@@ -260,6 +264,7 @@ pub mod test {
 
         // Completing slots should remove them from the repair list
         repairs = vec![];
+        slot_meta_cache = HashMap::default();
         let completed_shreds: Vec<Shred> = [0, 2, 4, 6]
             .iter()
             .map(|slot| {
@@ -281,6 +286,7 @@ pub mod test {
         get_best_repair_shreds(
             &heaviest_subtree_fork_choice,
             &blockstore,
+            &mut slot_meta_cache,
             &mut repairs,
             4,
             &HashSet::default(),
@@ -296,10 +302,12 @@ pub mod test {
         // Adding incomplete children with higher weighted parents, even if
         // the parents are complete should still be repaired
         repairs = vec![];
+        slot_meta_cache = HashMap::default();
         blockstore.add_tree(tr(2) / (tr(8)), true, false, 2, Hash::default());
         get_best_repair_shreds(
             &heaviest_subtree_fork_choice,
             &blockstore,
+            &mut slot_meta_cache,
             &mut repairs,
             4,
             &HashSet::default(),
@@ -320,9 +328,11 @@ pub mod test {
         // 4 again when the Unvisited(2) event happens
         blockstore.add_tree(tr(2) / (tr(6) / tr(7)), true, false, 2, Hash::default());
         let mut repairs = vec![];
+        let mut slot_meta_cache = HashMap::default();
         get_best_repair_shreds(
             &heaviest_subtree_fork_choice,
             &blockstore,
+            &mut slot_meta_cache,
             &mut repairs,
             std::usize::MAX,
             &HashSet::default(),
@@ -344,11 +354,13 @@ pub mod test {
         // Adding slots to ignore should remove them from the repair set, but
         // should not remove their children
         let mut repairs = vec![];
+        let mut slot_meta_cache = HashMap::default();
         let mut ignore_set: HashSet<Slot> = vec![1, 3].into_iter().collect();
         let last_shred = blockstore.meta(0).unwrap().unwrap().received;
         get_best_repair_shreds(
             &heaviest_subtree_fork_choice,
             &blockstore,
+            &mut slot_meta_cache,
             &mut repairs,
             std::usize::MAX,
             &ignore_set,
@@ -364,11 +376,13 @@ pub mod test {
         // Adding slot 2 to ignore should not remove its unexplored children from
         // the repair set
         repairs = vec![];
+        slot_meta_cache = HashMap::default();
         blockstore.add_tree(tr(2) / (tr(6) / tr(7)), true, false, 2, Hash::default());
         ignore_set.insert(2);
         get_best_repair_shreds(
             &heaviest_subtree_fork_choice,
             &blockstore,
+            &mut slot_meta_cache,
             &mut repairs,
             std::usize::MAX,
             &ignore_set,
@@ -385,9 +399,11 @@ pub mod test {
         // child 7 from the repair set
         repairs = vec![];
         ignore_set.insert(6);
+        slot_meta_cache = HashMap::default();
         get_best_repair_shreds(
             &heaviest_subtree_fork_choice,
             &blockstore,
+            &mut slot_meta_cache,
             &mut repairs,
             std::usize::MAX,
             &ignore_set,

ledger/src/blockstore_meta.rs

@@ -192,6 +192,14 @@ impl SlotMeta {
         self.consumed == self.last_index + 1
     }
 
+    pub fn known_last_index(&self) -> Option<u64> {
+        if self.last_index == std::u64::MAX {
+            None
+        } else {
+            Some(self.last_index)
+        }
+    }
+
     pub fn is_parent_set(&self) -> bool {
         self.parent_slot != std::u64::MAX
     }