Add more slot update notifications (#15734)

* Add more slot update notifications

* fix merge

* Address feedback and add integration test

* switch to datapoint

* remove unused shred method

* fix clippy

* new thread for rpc completed slots

* remove extra constant

* fixes

* rely on channel closing

* fix check

core/src/cluster_slots_service.rs

@@ -6,9 +6,9 @@ use solana_sdk::{clock::Slot, pubkey::Pubkey};
 use std::{
     sync::{
         atomic::{AtomicBool, Ordering},
+        mpsc::RecvTimeoutError,
         {Arc, RwLock},
     },
-    thread::sleep,
     thread::{self, Builder, JoinHandle},
     time::{Duration, Instant},
 };
@@ -79,6 +79,14 @@ impl ClusterSlotsService {
             if exit.load(Ordering::Relaxed) {
                 break;
             }
+            let slots = match completed_slots_receiver.recv_timeout(Duration::from_millis(200)) {
+                Ok(slots) => Some(slots),
+                Err(RecvTimeoutError::Timeout) => None,
+                Err(RecvTimeoutError::Disconnected) => {
+                    warn!("Cluster slots service - sender disconnected");
+                    break;
+                }
+            };
             let new_root = bank_forks.read().unwrap().root();
             let id = cluster_info.id();
             let mut lowest_slot_elapsed = Measure::start("lowest_slot_elapsed");
@@ -87,7 +95,9 @@ impl ClusterSlotsService {
             lowest_slot_elapsed.stop();
             let mut update_completed_slots_elapsed =
                 Measure::start("update_completed_slots_elapsed");
-            Self::update_completed_slots(&completed_slots_receiver, &cluster_info);
+            if let Some(slots) = slots {
+                Self::update_completed_slots(slots, &completed_slots_receiver, &cluster_info);
+            }
             cluster_slots.update(new_root, &cluster_info, &bank_forks);
             update_completed_slots_elapsed.stop();
 
@@ -113,20 +123,20 @@ impl ClusterSlotsService {
                 cluster_slots_service_timing = ClusterSlotsServiceTiming::default();
                 last_stats = Instant::now();
             }
-            sleep(Duration::from_millis(200));
         }
     }
 
     fn update_completed_slots(
+        mut slots: Vec<Slot>,
         completed_slots_receiver: &CompletedSlotsReceiver,
         cluster_info: &ClusterInfo,
     ) {
-        let mut slots: Vec<Slot> = vec![];
         while let Ok(mut more) = completed_slots_receiver.try_recv() {
             slots.append(&mut more);
         }
         #[allow(clippy::stable_sort_primitive)]
         slots.sort();
+
         if !slots.is_empty() {
             cluster_info.push_epoch_slots(&slots);
         }