gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

periodically report sigverify_stage stats (#19674)

Browse Source
This commit is contained in:
Jeff Biseda
2021-09-21 10:37:58 -07:00
committed by GitHub
parent 7f3d445af5
commit 640e93187c
4 changed files with 126 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
core/src/sigverify_stage.rs
View File

@ -8,14 +8,18 @@
use crate::sigverify;
use crossbeam_channel::{SendError, Sender as CrossbeamSender};
use solana_measure::measure::Measure;
use solana_metrics::datapoint_debug;
use solana_perf::packet::Packets;
use solana_perf::perf_libs;
use solana_sdk::timing;
use solana_streamer::streamer::{self, PacketReceiver, StreamerError};
use std::sync::mpsc::{Receiver, RecvTimeoutError};
use std::sync::{Arc, Mutex};
use std::thread::{self, Builder, JoinHandle};
use std::{
sync::{
mpsc::{Receiver, RecvTimeoutError},
Arc, Mutex,
},
thread::{self, Builder, JoinHandle},
time::Instant,
};
use thiserror::Error;
const RECV_BATCH_MAX_CPU: usize = 1_000;
@ -43,6 +47,82 @@ pub trait SigVerifier {
#[derive(Default, Clone)]
pub struct DisabledSigVerifier {}
#[derive(Default)]
struct SigVerifierStats {
recv_batches_us_hist: histogram::Histogram, // time to call recv_batch
verify_batches_pp_us_hist: histogram::Histogram, // per-packet time to call verify_batch
batches_hist: histogram::Histogram, // number of Packets structures per verify call
packets_hist: histogram::Histogram, // number of packets per verify call
total_batches: usize,
total_packets: usize,
}
impl SigVerifierStats {
fn report(&self) {
datapoint_info!(
"sigverify_stage-total_verify_time",
(
"recv_batches_us_90pct",
self.recv_batches_us_hist.percentile(90.0).unwrap_or(0),
i64
),
(
"recv_batches_us_min",
self.recv_batches_us_hist.minimum().unwrap_or(0),
i64
),
(
"recv_batches_us_max",
self.recv_batches_us_hist.maximum().unwrap_or(0),
i64
),
(
"recv_batches_us_mean",
self.recv_batches_us_hist.mean().unwrap_or(0),
i64
),
(
"verify_batches_pp_us_90pct",
self.verify_batches_pp_us_hist.percentile(90.0).unwrap_or(0),
i64
),
(
"verify_batches_pp_us_min",
self.verify_batches_pp_us_hist.minimum().unwrap_or(0),
i64
),
(
"verify_batches_pp_us_max",
self.verify_batches_pp_us_hist.maximum().unwrap_or(0),
i64
),
(
"verify_batches_pp_us_mean",
self.verify_batches_pp_us_hist.mean().unwrap_or(0),
i64
),
(
"batches_90pct",
self.batches_hist.percentile(90.0).unwrap_or(0),
i64
),
("batches_min", self.batches_hist.minimum().unwrap_or(0), i64),
("batches_max", self.batches_hist.maximum().unwrap_or(0), i64),
("batches_mean", self.batches_hist.mean().unwrap_or(0), i64),
(
"packets_90pct",
self.packets_hist.percentile(90.0).unwrap_or(0),
i64
),
("packets_min", self.packets_hist.minimum().unwrap_or(0), i64),
("packets_max", self.packets_hist.maximum().unwrap_or(0), i64),
("packets_mean", self.packets_hist.mean().unwrap_or(0), i64),
("total_batches", self.total_batches, i64),
("total_packets", self.total_packets, i64),
);
}
}
impl SigVerifier for DisabledSigVerifier {
fn verify_batch(&self, mut batch: Vec<Packets>) -> Vec<Packets> {
sigverify::ed25519_verify_disabled(&mut batch);
@ -66,8 +146,9 @@ impl SigVerifyStage {
sendr: &CrossbeamSender<Vec<Packets>>,
id: usize,
verifier: &T,
stats: &mut SigVerifierStats,
) -> Result<()> {
let (batch, len, recv_time) = streamer::recv_batch(
let (batch, num_packets, recv_duration) = streamer::recv_batch(
&recvr.lock().expect("'recvr' lock in fn verifier"),
if perf_libs::api().is_some() {
RECV_BATCH_MAX_GPU
@ -76,14 +157,15 @@ impl SigVerifyStage {
},
)?;
let mut verify_batch_time = Measure::start("sigverify_batch_time");
let batch_len = batch.len();
debug!(
"@{:?} verifier: verifying: {} id: {}",
timing::timestamp(),
len,
num_packets,
id
);
let mut verify_batch_time = Measure::start("sigverify_batch_time");
sendr.send(verifier.verify_batch(batch))?;
verify_batch_time.stop();
@ -93,17 +175,22 @@ impl SigVerifyStage {
batch_len,
verify_batch_time.as_ms(),
id,
len,
(len as f32 / verify_batch_time.as_s())
num_packets,
(num_packets as f32 / verify_batch_time.as_s())
);
datapoint_debug!(
"sigverify_stage-total_verify_time",
("num_batches", batch_len, i64),
("num_packets", len, i64),
("verify_time_ms", verify_batch_time.as_ms(), i64),
("recv_time", recv_time, i64),
);
stats
.recv_batches_us_hist
.increment(recv_duration.as_micros() as u64)
.unwrap();
stats
.verify_batches_pp_us_hist
.increment(verify_batch_time.as_us() / (num_packets as u64))
.unwrap();
stats.batches_hist.increment(batch_len as u64).unwrap();
stats.packets_hist.increment(num_packets as u64).unwrap();
stats.total_batches += batch_len;
stats.total_packets += num_packets;
Ok(())
}
@ -115,10 +202,18 @@ impl SigVerifyStage {
verifier: &T,
) -> JoinHandle<()> {
let verifier = verifier.clone();
let mut stats = SigVerifierStats::default();
let mut last_print = Instant::now();
Builder::new()
.name(format!("solana-verifier-{}", id))
.spawn(move || loop {
if let Err(e) = Self::verifier(&packet_receiver, &verified_sender, id, &verifier) {
if let Err(e) = Self::verifier(
&packet_receiver,
&verified_sender,
id,
&verifier,
&mut stats,
) {
match e {
SigVerifyServiceError::Streamer(StreamerError::RecvTimeout(
RecvTimeoutError::Disconnected,
@ -132,6 +227,11 @@ impl SigVerifyStage {
_ => error!("{:?}", e),
}
}
if last_print.elapsed().as_secs() > 2 {
stats.report();
stats = SigVerifierStats::default();
last_print = Instant::now();
}
})
.unwrap()
}