Poh timing service (#23736)

* initial work for poh timing report service

* add poh_timing_report_service to validator

* fix comments

* clippy

* imrove test coverage

* delete record when complete

* rename shred full to slot full.

* debug logging

* fix slot full

* remove debug comments

* adding fmt trait

* derive default

* default for poh timing reporter

* better comments

* remove commented code

* fix test

* more test fixes

* delete timestamps for slot that are older than root_slot

* debug log

* record poh start end in bank reset

* report full to start time instead

* fix poh slot offset

* report poh start for normal ticks

* fix typo

* refactor out poh point report fn

* rename

* optimize delete - delete only when last_root changed

* change log level to trace

* convert if to match

* remove redudant check

* fix SlotPohTiming comments

* review feedback on poh timing reporter

* review feedback on poh_recorder

* add test case for out-of-order arrival of timing points and incomplete timing points

* refactor poh_timing_points into its own mod

* remove option for poh_timing_report service

* move poh_timing_point_sender to constructor

* clippy

* better comments

* more clippy

* more clippy

* add slot poh timing point macro

* clippy

* assert in test

* comments and display fmt

* fix check

* assert format

* revise comments

* refactor

* extrac send fn

* revert reporting_poh_timing_point

* align loggin

* small refactor

* move type declaration to the top of the module

* replace macro with constructor

* clippy: remove redundant closure

* review comments

* simplify poh timing point creation

Co-authored-by: Haoran Yi <hyi@Haorans-MacBook-Air.local>

metrics/src/lib.rs

@@ -2,6 +2,7 @@
 pub mod counter;
 pub mod datapoint;
 mod metrics;
+pub mod poh_timing_point;
 pub use crate::metrics::{flush, query, set_host_id, set_panic_hook, submit};
 use std::sync::Arc;
 

metrics/src/poh_timing_point.rs

@@ -0,0 +1,173 @@
+//! A poh_timing_point module
+
+use {
+    crossbeam_channel::{Receiver, Sender},
+    log::*,
+    solana_sdk::clock::Slot,
+    std::fmt,
+};
+
+/// Receiver of SlotPohTimingInfo from the channel
+pub type PohTimingReceiver = Receiver<SlotPohTimingInfo>;
+
+/// Sender of SlotPohTimingInfo to the channel
+pub type PohTimingSender = Sender<SlotPohTimingInfo>;
+
+/// PohTimingPoint. Each TimingPoint is annotated with a timestamp in milliseconds.
+#[derive(Debug, Clone, PartialEq)]
+pub enum PohTimingPoint {
+    PohSlotStart(u64),
+    PohSlotEnd(u64),
+    FullSlotReceived(u64),
+}
+
+impl fmt::Display for PohTimingPoint {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            PohTimingPoint::PohSlotStart(t) => write!(f, "poh_start={}", t),
+            PohTimingPoint::PohSlotEnd(t) => write!(f, "poh_end  ={}", t),
+            PohTimingPoint::FullSlotReceived(t) => write!(f, "poh_full ={}", t),
+        }
+    }
+}
+
+/// SlotPohTimingInfo. This struct is sent to channel and received by
+/// poh_timing_report service.
+#[derive(Clone, Debug)]
+pub struct SlotPohTimingInfo {
+    /// current slot
+    pub slot: Slot,
+    /// root slot
+    pub root_slot: Option<Slot>,
+    /// timing event
+    pub timing_point: PohTimingPoint,
+}
+
+impl fmt::Display for SlotPohTimingInfo {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "PohTimingPoint: {}, slot={}, root_slot={}",
+            self.timing_point,
+            self.slot,
+            self.root_slot.unwrap_or(0),
+        )
+    }
+}
+
+impl SlotPohTimingInfo {
+    /// create slot start poh timing point
+    pub fn new_slot_start_poh_time_point(
+        slot: Slot,
+        root_slot: Option<Slot>,
+        timestamp: u64,
+    ) -> SlotPohTimingInfo {
+        SlotPohTimingInfo {
+            slot,
+            root_slot,
+            timing_point: PohTimingPoint::PohSlotStart(timestamp),
+        }
+    }
+
+    /// create slot end poh timing point
+    pub fn new_slot_end_poh_time_point(
+        slot: Slot,
+        root_slot: Option<Slot>,
+        timestamp: u64,
+    ) -> SlotPohTimingInfo {
+        SlotPohTimingInfo {
+            slot,
+            root_slot,
+            timing_point: PohTimingPoint::PohSlotEnd(timestamp),
+        }
+    }
+
+    /// create slot full poh timing point
+    pub fn new_slot_full_poh_time_point(
+        slot: Slot,
+        root_slot: Option<Slot>,
+        timestamp: u64,
+    ) -> SlotPohTimingInfo {
+        SlotPohTimingInfo {
+            slot,
+            root_slot,
+            timing_point: PohTimingPoint::FullSlotReceived(timestamp),
+        }
+    }
+}
+
+/// send poh timing to channel
+pub fn send_poh_timing_point(sender: &PohTimingSender, slot_timing: SlotPohTimingInfo) {
+    trace!("{}", slot_timing);
+    if let Err(e) = sender.try_send(slot_timing) {
+        info!("failed to send slot poh timing {:?}", e);
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    #[test]
+    fn test_poh_timing_point() {
+        // create slot start with root
+        let p = SlotPohTimingInfo::new_slot_start_poh_time_point(100, Some(101), 100);
+        assert!(p.slot == 100);
+        assert_eq!(p.root_slot, Some(101));
+        assert_eq!(p.timing_point, PohTimingPoint::PohSlotStart(100));
+        assert_eq!(
+            format!("{}", p),
+            "PohTimingPoint: poh_start=100, slot=100, root_slot=101"
+        );
+
+        // create slot start without root
+        let p = SlotPohTimingInfo::new_slot_start_poh_time_point(100, None, 100);
+        assert!(p.slot == 100);
+        assert_eq!(p.root_slot, None);
+        assert_eq!(p.timing_point, PohTimingPoint::PohSlotStart(100));
+        assert_eq!(
+            format!("{}", p),
+            "PohTimingPoint: poh_start=100, slot=100, root_slot=0"
+        );
+
+        // create slot end with root
+        let p = SlotPohTimingInfo::new_slot_end_poh_time_point(100, Some(101), 100);
+        assert!(p.slot == 100);
+        assert_eq!(p.root_slot, Some(101));
+        assert_eq!(p.timing_point, PohTimingPoint::PohSlotEnd(100));
+        assert_eq!(
+            format!("{}", p),
+            "PohTimingPoint: poh_end  =100, slot=100, root_slot=101"
+        );
+
+        // create slot end without root
+        let p = SlotPohTimingInfo::new_slot_end_poh_time_point(100, None, 100);
+        assert!(p.slot == 100);
+        assert_eq!(p.root_slot, None);
+        assert_eq!(p.timing_point, PohTimingPoint::PohSlotEnd(100));
+        assert_eq!(
+            format!("{}", p),
+            "PohTimingPoint: poh_end  =100, slot=100, root_slot=0"
+        );
+
+        // create slot full with root
+        let p = SlotPohTimingInfo::new_slot_full_poh_time_point(100, Some(101), 100);
+        assert!(p.slot == 100);
+        assert_eq!(p.root_slot, Some(101));
+        assert_eq!(p.timing_point, PohTimingPoint::FullSlotReceived(100));
+        assert_eq!(
+            format!("{}", p),
+            "PohTimingPoint: poh_full =100, slot=100, root_slot=101"
+        );
+
+        // create slot full without root
+        let p = SlotPohTimingInfo::new_slot_full_poh_time_point(100, None, 100);
+        assert!(p.slot == 100);
+        assert_eq!(p.root_slot, None);
+        assert_eq!(p.timing_point, PohTimingPoint::FullSlotReceived(100));
+
+        assert_eq!(
+            format!("{}", p),
+            "PohTimingPoint: poh_full =100, slot=100, root_slot=0"
+        );
+    }
+}