gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor

Browse Source
This commit is contained in:
Michael Vines
2020-03-09 22:02:40 -07:00
parent a0ed3261c9
commit 23c5bb17c7
1 changed files with 101 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

166
watchtower/src/main.rs
View File

@ -10,11 +10,18 @@ use solana_clap_utils::{
input_validators::{is_pubkey_or_keypair, is_url}, input_validators::{is_pubkey_or_keypair, is_url},
}; };
use solana_cli_config::{Config, CONFIG_FILE}; use solana_cli_config::{Config, CONFIG_FILE};
use solana_client::rpc_client::RpcClient; use solana_client::{rpc_client::RpcClient, rpc_response::RpcVoteAccountStatus};
use solana_metrics::{datapoint_error, datapoint_info}; use solana_metrics::{datapoint_error, datapoint_info};
use solana_sdk::native_token::lamports_to_sol; use solana_sdk::{hash::Hash, native_token::lamports_to_sol};
use std::{error, io, thread::sleep, time::Duration}; use std::{error, io, thread::sleep, time::Duration};
fn get_cluster_info(rpc_client: &RpcClient) -> io::Result<(u64, Hash, RpcVoteAccountStatus)> {
let transaction_count = rpc_client.get_transaction_count()?;
let recent_blockhash = rpc_client.get_recent_blockhash()?.0;
let vote_accounts = rpc_client.get_vote_accounts()?;
Ok((transaction_count, recent_blockhash, vote_accounts))
}
fn main() -> Result<(), Box<dyn error::Error>> { fn main() -> Result<(), Box<dyn error::Error>> {
let matches = App::new(crate_name!()) let matches = App::new(crate_name!())
.about(crate_description!()) .about(crate_description!())
@ -95,59 +102,21 @@ fn main() -> Result<(), Box<dyn error::Error>> {
let notifier = Notifier::new(); let notifier = Notifier::new();
let mut last_transaction_count = 0; let mut last_transaction_count = 0;
let mut last_check_notification_sent = false; let mut last_recent_blockhash = Hash::default();
let mut last_notification_msg = String::from(""); let mut last_notification_msg = "".into();
loop {
let mut notify_msg = String::from("solana-watchtower: undefined error");
let ok = rpc_client
.get_transaction_count()
.and_then(|transaction_count| {
info!("Current transaction count: {}", transaction_count);
if transaction_count > last_transaction_count { loop {
last_transaction_count = transaction_count; let failure = match get_cluster_info(&rpc_client) {
Ok(true) Ok((transaction_count, recent_blockhash, vote_accounts)) => {
} else { info!("Current transaction count: {}", transaction_count);
Err(io::Error::new( info!("Recent blockhash: {}", recent_blockhash);
io::ErrorKind::Other, info!("Current validator count: {}", vote_accounts.current.len());
format!( info!(
"Transaction count is not advancing: {} <= {}", "Delinquent validator count: {}",
transaction_count, last_transaction_count vote_accounts.delinquent.len()
),
))
}
})
.unwrap_or_else(|err| {
notify_msg = format!("solana-watchtower: {}", err.to_string());
datapoint_error!(
"watchtower-sanity-failure",
("test", "transaction-count", String),
("err", err.to_string(), String)
); );
false
}) let mut failures = vec![];
&& rpc_client
.get_recent_blockhash()
.and_then(|(blockhash, _fee_calculator)| {
info!("Current blockhash: {}", blockhash);
rpc_client.get_new_blockhash(&blockhash)
})
.and_then(|(blockhash, _fee_calculator)| {
info!("New blockhash: {}", blockhash);
Ok(true)
})
.unwrap_or_else(|err| {
notify_msg = format!("solana-watchtower: {}", err.to_string());
datapoint_error!(
"watchtower-sanity-failure",
("test", "blockhash", String),
("err", err.to_string(), String)
);
false
})
&& rpc_client
.get_vote_accounts()
.and_then(|vote_accounts| {
let total_current_stake = vote_accounts let total_current_stake = vote_accounts
.current .current
@ -168,20 +137,33 @@ fn main() -> Result<(), Box<dyn error::Error>> {
lamports_to_sol(total_delinquent_stake) lamports_to_sol(total_delinquent_stake)
); );
info!("Current validator count: {}", vote_accounts.current.len()); if transaction_count > last_transaction_count {
info!( last_transaction_count = transaction_count;
"Delinquent validator count: {}", } else {
vote_accounts.delinquent.len() failures.push((
); "transaction-count",
format!(
"Transaction count is not advancing: {} <= {}",
transaction_count, last_transaction_count
),
));
}
if recent_blockhash != last_recent_blockhash {
last_recent_blockhash = recent_blockhash;
} else {
failures.push((
"recent-blockhash",
format!("Unable to get new blockhash: {}", recent_blockhash),
));
}
if validator_identity_pubkeys.is_empty() { if validator_identity_pubkeys.is_empty() {
if vote_accounts.delinquent.is_empty() { if !vote_accounts.delinquent.is_empty() {
Ok(true) failures.push((
} else { "delinquent",
Err(io::Error::new(
io::ErrorKind::Other,
format!("{} delinquent validators", vote_accounts.delinquent.len()), format!("{} delinquent validators", vote_accounts.delinquent.len()),
)) ));
} }
} else { } else {
let mut errors = vec![]; let mut errors = vec![];
@ -201,45 +183,37 @@ fn main() -> Result<(), Box<dyn error::Error>> {
} }
} }
if errors.is_empty() { if !errors.is_empty() {
Ok(true) failures.push(("delinquent", errors.join(",")));
} else {
Err(io::Error::new(io::ErrorKind::Other, errors.join(",")))
} }
} }
})
.unwrap_or_else(|err| { failures.into_iter().next() // Only report the first failure if any
notify_msg = format!("solana-watchtower: {}", err.to_string()); }
Err(err) => Some(("rpc", err.to_string())),
};
datapoint_info!("watchtower-sanity", ("ok", failure.is_none(), bool));
if let Some((failure_test_name, failure_error_message)) = &failure {
let notification_msg = format!(
"solana-watchtower: Error: {}: {}",
failure_test_name, failure_error_message
);
if !no_duplicate_notifications || last_notification_msg != notification_msg {
notifier.send(&notification_msg);
}
datapoint_error!( datapoint_error!(
"watchtower-sanity-failure", "watchtower-sanity-failure",
("test", "delinquent-validators", String), ("test", failure_test_name, String),
("err", err.to_string(), String) ("err", failure_error_message, String)
); );
false last_notification_msg = notification_msg;
});
datapoint_info!("watchtower-sanity", ("ok", ok, bool));
if !ok {
last_check_notification_sent = true;
if no_duplicate_notifications {
if last_notification_msg != notify_msg {
notifier.send(&notify_msg);
last_notification_msg = notify_msg;
} else { } else {
datapoint_info!( if !last_notification_msg.is_empty() {
"watchtower-sanity", info!("All clear");
("Suppressing duplicate notification", ok, bool) notifier.send("solana-watchtower: All clear");
);
} }
} else { last_notification_msg = "".into();
notifier.send(&notify_msg);
}
} else {
if last_check_notification_sent {
notifier.send("solana-watchtower: All Clear");
}
last_check_notification_sent = false;
last_notification_msg = String::from("");
} }
sleep(interval); sleep(interval);
} }