More failure codepath tracing (#15246)

ledger-tool/src/main.rs

@@ -2256,6 +2256,7 @@ fn main() {
                             point_value: Option<PointValue>,
                             old_credits_observed: Option<u64>,
                             new_credits_observed: Option<u64>,
+                            skipped_reasons: String,
                         }
                         use solana_stake_program::stake_state::InflationPointCalculationEvent;
                         let mut stake_calcuration_details: HashMap<Pubkey, CalculationDetail> =
@@ -2313,7 +2314,7 @@ fn main() {
                                         new_credits_observed,
                                     ) => {
                                         detail.old_credits_observed = Some(*old_credits_observed);
-                                        detail.new_credits_observed = Some(*new_credits_observed);
+                                        detail.new_credits_observed = *new_credits_observed;
                                     }
                                     InflationPointCalculationEvent::Delegation(
                                         delegation,
@@ -2328,6 +2329,13 @@ fn main() {
                                                 Some(delegation.deactivation_epoch);
                                         }
                                     }
+                                    InflationPointCalculationEvent::Skipped(skipped_reason) => {
+                                        if detail.skipped_reasons.is_empty() {
+                                            detail.skipped_reasons = format!("{:?}", skipped_reason);
+                                        } else {
+                                            detail.skipped_reasons += &format!("/{:?}", skipped_reason);
+                                        }
+                                    }
                                 }
                             }
                         };

programs/stake/src/stake_state.rs

@@ -33,6 +33,21 @@ pub enum StakeState {
     RewardsPool,
 }
 
+#[derive(Debug)]
+pub enum SkippedReason {
+    ZeroPoints,
+    ZeroPointValue,
+    ZeroReward,
+    ZeroCreditsAndReturnZero,
+    ZeroCreditsAndReturnCurrent,
+}
+
+impl From<SkippedReason> for InflationPointCalculationEvent {
+    fn from(reason: SkippedReason) -> Self {
+        InflationPointCalculationEvent::Skipped(reason)
+    }
+}
+
 #[derive(Debug)]
 pub enum InflationPointCalculationEvent {
     CalculatedPoints(u64, u128, u128, u128),
@@ -41,7 +56,8 @@ pub enum InflationPointCalculationEvent {
     RentExemptReserve(u64),
     Delegation(Delegation, Pubkey),
     Commission(u8),
-    CreditsObserved(u64, u64),
+    CreditsObserved(u64, Option<u64>),
+    Skipped(SkippedReason),
 }
 
 pub(crate) fn null_tracer() -> Option<impl FnMut(&InflationPointCalculationEvent)> {
@@ -543,6 +559,12 @@ impl Stake {
         inflation_point_calc_tracer: &mut Option<impl FnMut(&InflationPointCalculationEvent)>,
         fix_stake_deactivate: bool,
     ) -> Option<(u64, u64)> {
+        if let Some(inflation_point_calc_tracer) = inflation_point_calc_tracer {
+            inflation_point_calc_tracer(&InflationPointCalculationEvent::CreditsObserved(
+                self.credits_observed,
+                None,
+            ));
+        }
         self.calculate_rewards(
             point_value,
             vote_state,
@@ -554,7 +576,7 @@ impl Stake {
             if let Some(inflation_point_calc_tracer) = inflation_point_calc_tracer {
                 inflation_point_calc_tracer(&InflationPointCalculationEvent::CreditsObserved(
                     self.credits_observed,
-                    credits_observed,
+                    Some(credits_observed),
                 ));
             }
             self.credits_observed = credits_observed;
@@ -592,8 +614,14 @@ impl Stake {
         // if there is no newer credits since observed, return no point
         if new_vote_state.credits() <= self.credits_observed {
             if fix_stake_deactivate {
+                if let Some(inflation_point_calc_tracer) = inflation_point_calc_tracer {
+                    inflation_point_calc_tracer(&SkippedReason::ZeroCreditsAndReturnCurrent.into());
+                }
                 return (0, self.credits_observed);
             } else {
+                if let Some(inflation_point_calc_tracer) = inflation_point_calc_tracer {
+                    inflation_point_calc_tracer(&SkippedReason::ZeroCreditsAndReturnZero.into());
+                }
                 return (0, 0);
             }
         }
@@ -671,7 +699,16 @@ impl Stake {
             return Some((0, 0, credits_observed));
         }
 
-        if points == 0 || point_value.points == 0 {
+        if points == 0 {
+            if let Some(inflation_point_calc_tracer) = inflation_point_calc_tracer {
+                inflation_point_calc_tracer(&SkippedReason::ZeroPoints.into());
+            }
+            return None;
+        }
+        if point_value.points == 0 {
+            if let Some(inflation_point_calc_tracer) = inflation_point_calc_tracer {
+                inflation_point_calc_tracer(&SkippedReason::ZeroPointValue.into());
+            }
             return None;
         }
 
@@ -685,6 +722,9 @@ impl Stake {
 
         // don't bother trying to split if fractional lamports got truncated
         if rewards == 0 {
+            if let Some(inflation_point_calc_tracer) = inflation_point_calc_tracer {
+                inflation_point_calc_tracer(&SkippedReason::ZeroReward.into());
+            }
             return None;
         }
         let (voter_rewards, staker_rewards, is_split) = vote_state.commission_split(rewards);