adds more metrics to blockstore insert shreds stats (#20701)

ledger/src/blockstore.rs

@@ -46,7 +46,7 @@ use {
         borrow::Cow,
         cell::RefCell,
         cmp,
-        collections::{BTreeMap, HashMap, HashSet},
+        collections::{hash_map::Entry as HashMapEntry, BTreeMap, HashMap, HashSet},
         convert::TryInto,
         fs,
         io::{Error as IoError, ErrorKind},
@@ -213,11 +213,19 @@ pub struct BlockstoreInsertionMetrics {
     pub num_inserted: u64,
     pub num_repair: u64,
     pub num_recovered: usize,
+    num_recovered_blockstore_error: usize,
     pub num_recovered_inserted: usize,
     pub num_recovered_failed_sig: usize,
     pub num_recovered_failed_invalid: usize,
     pub num_recovered_exists: usize,
     pub index_meta_time: u64,
+    num_data_shreds_exists: usize,
+    num_data_shreds_invalid: usize,
+    num_data_shreds_blockstore_error: usize,
+    num_coding_shreds_exists: usize,
+    num_coding_shreds_invalid: usize,
+    num_coding_shreds_invalid_erasure_config: usize,
+    num_coding_shreds_inserted: usize,
 }
 
 impl SlotMetaWorkingSetEntry {
@@ -277,6 +285,38 @@ impl BlockstoreInsertionMetrics {
                 self.num_recovered_exists as i64,
                 i64
             ),
+            (
+                "num_recovered_blockstore_error",
+                self.num_recovered_blockstore_error,
+                i64
+            ),
+            ("num_data_shreds_exists", self.num_data_shreds_exists, i64),
+            ("num_data_shreds_invalid", self.num_data_shreds_invalid, i64),
+            (
+                "num_data_shreds_blockstore_error",
+                self.num_data_shreds_blockstore_error,
+                i64
+            ),
+            (
+                "num_coding_shreds_exists",
+                self.num_coding_shreds_exists,
+                i64
+            ),
+            (
+                "num_coding_shreds_invalid",
+                self.num_coding_shreds_invalid,
+                i64
+            ),
+            (
+                "num_coding_shreds_invalid_erasure_config",
+                self.num_coding_shreds_invalid_erasure_config,
+                i64
+            ),
+            (
+                "num_coding_shreds_inserted",
+                self.num_coding_shreds_inserted,
+                i64
+            ),
         );
     }
 }
@@ -838,7 +878,7 @@ impl Blockstore {
                 } else {
                     ShredSource::Turbine
                 };
-                if let Ok(completed_data_sets) = self.check_insert_data_shred(
+                match self.check_insert_data_shred(
                     shred,
                     &mut erasure_metas,
                     &mut index_working_set,
@@ -851,10 +891,18 @@ impl Blockstore {
                     leader_schedule,
                     shred_source,
                 ) {
+                    Err(InsertDataShredError::Exists) => metrics.num_data_shreds_exists += 1,
+                    Err(InsertDataShredError::InvalidShred) => metrics.num_data_shreds_invalid += 1,
+                    Err(InsertDataShredError::BlockstoreError(err)) => {
+                        metrics.num_data_shreds_blockstore_error += 1;
+                        error!("blockstore error: {}", err);
+                    }
+                    Ok(completed_data_sets) => {
                         newly_completed_data_sets.extend(completed_data_sets);
                         inserted_indices.push(i);
                         metrics.num_inserted += 1;
                     }
+                };
             } else if shred.is_code() {
                 self.check_cache_coding_shred(
                     shred,
@@ -865,6 +913,7 @@ impl Blockstore {
                     handle_duplicate,
                     is_trusted,
                     is_repaired,
+                    metrics,
                 );
             } else {
                 panic!("There should be no other case");
@@ -914,7 +963,11 @@ impl Blockstore {
                             metrics.num_recovered_failed_invalid += 1;
                             None
                         }
-                        Err(InsertDataShredError::BlockstoreError(_)) => None,
+                        Err(InsertDataShredError::BlockstoreError(err)) => {
+                            metrics.num_recovered_blockstore_error += 1;
+                            error!("blockstore error: {}", err);
+                            None
+                        }
                         Ok(completed_data_sets) => {
                             newly_completed_data_sets.extend(completed_data_sets);
                             metrics.num_recovered_inserted += 1;
@@ -1059,6 +1112,7 @@ impl Blockstore {
             || shred1.coding_header.num_data_shreds != shred2.coding_header.num_data_shreds
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn check_cache_coding_shred<F>(
         &self,
         shred: Shred,
@@ -1069,6 +1123,7 @@ impl Blockstore {
         handle_duplicate: &F,
         is_trusted: bool,
         is_repaired: bool,
+        metrics: &mut BlockstoreInsertionMetrics,
     ) -> bool
     where
         F: Fn(Shred),
@@ -1086,11 +1141,13 @@ impl Blockstore {
 
         if !is_trusted {
             if index_meta.coding().is_present(shred_index) {
+                metrics.num_coding_shreds_exists += 1;
                 handle_duplicate(shred);
                 return false;
             }
 
             if !Blockstore::should_insert_coding_shred(&shred, &self.last_root) {
+                metrics.num_coding_shreds_invalid += 1;
                 return false;
             }
         }
@@ -1109,6 +1166,7 @@ impl Blockstore {
         });
 
         if erasure_config != erasure_meta.config {
+            metrics.num_coding_shreds_invalid_erasure_config += 1;
             let conflicting_shred = self.find_conflicting_coding_shred(
                 &shred,
                 slot,
@@ -1149,10 +1207,11 @@ impl Blockstore {
         // committed
         index_meta.coding_mut().set_present(shred_index, true);
 
-        just_received_coding_shreds
+        if let HashMapEntry::Vacant(entry) = just_received_coding_shreds.entry((slot, shred_index))
-            .entry((slot, shred_index))
+        {
-            .or_insert_with(|| shred);
+            metrics.num_coding_shreds_inserted += 1;
-
+            entry.insert(shred);
+        }
         true
     }
 
@@ -5665,6 +5724,7 @@ pub mod tests {
             },
             false,
             false,
+            &mut BlockstoreInsertionMetrics::default(),
         ));
 
         // insert again fails on dupe
@@ -5681,6 +5741,7 @@ pub mod tests {
             },
             false,
             false,
+            &mut BlockstoreInsertionMetrics::default(),
         ));
         assert_eq!(counter.load(Ordering::Relaxed), 1);
     }