Add cluster state verifier logging (#16330)

* Add cluster state verifier logging

* Add duplicate-slots iterator to ledger tool

core/src/cluster_slot_state_verifier.rs

@@ -239,6 +239,15 @@ pub(crate) fn check_slot_agrees_with_cluster(
     fork_choice: &mut HeaviestSubtreeForkChoice,
     slot_state_update: SlotStateUpdate,
 ) {
+    info!(
+        "check_slot_agrees_with_cluster()
+        slot: {},
+        root: {},
+        frozen_hash: {:?},
+        update: {:?}",
+        slot, root, frozen_hash, slot_state_update
+    );
+
     if slot <= root {
         return;
     }
@@ -252,6 +261,7 @@ pub(crate) fn check_slot_agrees_with_cluster(
 
     let frozen_hash = frozen_hash.unwrap();
     let gossip_duplicate_confirmed_hash = gossip_duplicate_confirmed_slots.get(&slot);
+
     let is_local_replay_duplicate_confirmed = progress.is_duplicate_confirmed(slot).expect("If the frozen hash exists, then the slot must exist in bank forks and thus in progress map");
     let cluster_duplicate_confirmed_hash = get_cluster_duplicate_confirmed_hash(
         slot,
@@ -273,6 +283,18 @@ pub(crate) fn check_slot_agrees_with_cluster(
     }
     let is_dead = progress.is_dead(slot).expect("If the frozen hash exists, then the slot must exist in bank forks and thus in progress map");
 
+    info!(
+        "check_slot_agrees_with_cluster() state
+        is_local_replay_duplicate_confirmed: {:?},
+        cluster_duplicate_confirmed_hash: {:?},
+        is_slot_duplicate: {:?},
+        is_dead: {:?}",
+        is_local_replay_duplicate_confirmed,
+        cluster_duplicate_confirmed_hash,
+        is_slot_duplicate,
+        is_dead,
+    );
+
     let state_handler = slot_state_update.to_handler();
     let state_changes = state_handler(
         slot,

core/src/heaviest_subtree_fork_choice.rs

@@ -471,6 +471,12 @@ impl HeaviestSubtreeForkChoice {
 
     fn mark_slot_valid(&mut self, valid_slot: Slot) {
         if let Some(fork_info) = self.fork_infos.get_mut(&valid_slot) {
+            if !fork_info.is_candidate {
+                info!(
+                    "marked previously invalid fork starting at slot: {} as valid",
+                    valid_slot
+                );
+            }
             fork_info.is_candidate = true;
         }
     }
@@ -683,6 +689,10 @@ impl ForkChoice for HeaviestSubtreeForkChoice {
     }
 
     fn mark_fork_invalid_candidate(&mut self, invalid_slot: Slot) {
+        info!(
+            "marking fork starting at slot: {} invalid candidate",
+            invalid_slot
+        );
         let fork_info = self.fork_infos.get_mut(&invalid_slot);
         if let Some(fork_info) = fork_info {
             if fork_info.is_candidate {

core/src/replay_stage.rs

@@ -934,7 +934,6 @@ impl ReplayStage {
         for (duplicate_slot, bank_hash) in duplicate_slots.into_iter().zip(bank_hashes.into_iter())
         {
             // WindowService should only send the signal once per slot
-
             check_slot_agrees_with_cluster(
                 duplicate_slot,
                 root_slot,