gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

watchtower: Add --monitor-active-stake flag (bp #8765) (#8769)

Browse Source 
automerge
This commit is contained in:
mergify[bot]
2020-03-10 13:27:19 -07:00
committed by GitHub
parent 38e661d7ba
commit 1f80346d97
1 changed files with 148 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

230
watchtower/src/main.rs
View File

@ -9,13 +9,20 @@ use solana_clap_utils::{
input_parsers::pubkeys_of,
input_validators::{is_pubkey_or_keypair, is_url},
};
use solana_cli_config::{Config, CONFIG_FILE};
use solana_client::rpc_client::RpcClient;
use solana_client::{rpc_client::RpcClient, rpc_response::RpcVoteAccountStatus};
use solana_metrics::{datapoint_error, datapoint_info};
use solana_sdk::native_token::lamports_to_sol;
use solana_sdk::{hash::Hash, native_token::lamports_to_sol};
use std::{error, io, thread::sleep, time::Duration};
fn main() -> Result<(), Box<dyn error::Error>> {
struct Config {
interval: Duration,
json_rpc_url: String,
validator_identity_pubkeys: Vec<String>,
no_duplicate_notifications: bool,
monitor_active_stake: bool,
}
fn get_config() -> Config {
let matches = App::new(crate_name!())
.about(crate_description!())
.version(solana_clap_utils::version!())
@ -27,7 +34,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
.takes_value(true)
.global(true)
.help("Configuration file to use");
if let Some(ref config_file) = *CONFIG_FILE {
if let Some(ref config_file) = *solana_cli_config::CONFIG_FILE {
arg.default_value(&config_file)
} else {
arg
@ -50,7 +57,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
.help("Wait interval seconds between checking the cluster"),
)
.arg(
Arg::with_name("validator_identitys")
Arg::with_name("validator_identities")
.long("validator-identity")
.value_name("VALIDATOR IDENTITY PUBKEY")
.takes_value(true)
@ -64,90 +71,81 @@ fn main() -> Result<(), Box<dyn error::Error>> {
.takes_value(false)
.help("Subsequent identical notifications will be suppressed"),
)
.arg(
Arg::with_name("monitor_active_stake")
.long("monitor-active-stake")
.takes_value(false)
.help("Alert when the current stake for the cluster drops below 80%"),
)
.get_matches();
let config = if let Some(config_file) = matches.value_of("config_file") {
Config::load(config_file).unwrap_or_default()
solana_cli_config::Config::load(config_file).unwrap_or_default()
} else {
Config::default()
solana_cli_config::Config::default()
};
let interval = Duration::from_secs(value_t_or_exit!(matches, "interval", u64));
let json_rpc_url =
value_t!(matches, "json_rpc_url", String).unwrap_or_else(|_| config.json_rpc_url);
let validator_identity_pubkeys: Vec<_> = pubkeys_of(&matches, "validator_identitys")
let validator_identity_pubkeys: Vec<_> = pubkeys_of(&matches, "validator_identities")
.unwrap_or_else(|| vec![])
.into_iter()
.map(|i| i.to_string())
.collect();
let no_duplicate_notifications = matches.is_present("no_duplicate_notifications");
let monitor_active_stake = matches.is_present("monitor_active_stake");
Config {
interval,
json_rpc_url,
validator_identity_pubkeys,
no_duplicate_notifications,
monitor_active_stake,
}
}
fn get_cluster_info(rpc_client: &RpcClient) -> io::Result<(u64, Hash, RpcVoteAccountStatus)> {
let transaction_count = rpc_client.get_transaction_count()?;
let recent_blockhash = rpc_client.get_recent_blockhash()?.0;
let vote_accounts = rpc_client.get_vote_accounts()?;
Ok((transaction_count, recent_blockhash, vote_accounts))
}
fn main() -> Result<(), Box<dyn error::Error>> {
let config = get_config();
solana_logger::setup_with_default("solana=info");
solana_metrics::set_panic_hook("watchtower");
info!("RPC URL: {}", json_rpc_url);
if !validator_identity_pubkeys.is_empty() {
info!("Monitored validators: {:?}", validator_identity_pubkeys);
info!("RPC URL: {}", config.json_rpc_url);
if !config.validator_identity_pubkeys.is_empty() {
info!(
"Monitored validators: {:?}",
config.validator_identity_pubkeys
);
}
let rpc_client = RpcClient::new(json_rpc_url);
let rpc_client = RpcClient::new(config.json_rpc_url);
let notifier = Notifier::new();
let mut last_transaction_count = 0;
let mut last_check_notification_sent = false;
let mut last_notification_msg = String::from("");
loop {
let mut notify_msg = String::from("solana-watchtower: undefined error");
let ok = rpc_client
.get_transaction_count()
.and_then(|transaction_count| {
info!("Current transaction count: {}", transaction_count);
let mut last_recent_blockhash = Hash::default();
let mut last_notification_msg = "".into();
if transaction_count > last_transaction_count {
last_transaction_count = transaction_count;
Ok(true)
} else {
Err(io::Error::new(
io::ErrorKind::Other,
format!(
"Transaction count is not advancing: {} <= {}",
transaction_count, last_transaction_count
),
))
}
})
.unwrap_or_else(|err| {
notify_msg = format!("solana-watchtower: {}", err.to_string());
datapoint_error!(
"watchtower-sanity-failure",
("test", "transaction-count", String),
("err", err.to_string(), String)
loop {
let failure = match get_cluster_info(&rpc_client) {
Ok((transaction_count, recent_blockhash, vote_accounts)) => {
info!("Current transaction count: {}", transaction_count);
info!("Recent blockhash: {}", recent_blockhash);
info!("Current validator count: {}", vote_accounts.current.len());
info!(
"Delinquent validator count: {}",
vote_accounts.delinquent.len()
);
false
})
&& rpc_client
.get_recent_blockhash()
.and_then(|(blockhash, _fee_calculator)| {
info!("Current blockhash: {}", blockhash);
rpc_client.get_new_blockhash(&blockhash)
})
.and_then(|(blockhash, _fee_calculator)| {
info!("New blockhash: {}", blockhash);
Ok(true)
})
.unwrap_or_else(|err| {
notify_msg = format!("solana-watchtower: {}", err.to_string());
datapoint_error!(
"watchtower-sanity-failure",
("test", "blockhash", String),
("err", err.to_string(), String)
);
false
})
&& rpc_client
.get_vote_accounts()
.and_then(|vote_accounts| {
let mut failures = vec![];
let total_current_stake = vote_accounts
.current
@ -168,24 +166,44 @@ fn main() -> Result<(), Box<dyn error::Error>> {
lamports_to_sol(total_delinquent_stake)
);
info!("Current validator count: {}", vote_accounts.current.len());
info!(
"Delinquent validator count: {}",
vote_accounts.delinquent.len()
);
if validator_identity_pubkeys.is_empty() {
if vote_accounts.delinquent.is_empty() {
Ok(true)
if transaction_count > last_transaction_count {
last_transaction_count = transaction_count;
} else {
Err(io::Error::new(
io::ErrorKind::Other,
failures.push((
"transaction-count",
format!(
"Transaction count is not advancing: {} <= {}",
transaction_count, last_transaction_count
),
));
}
if recent_blockhash != last_recent_blockhash {
last_recent_blockhash = recent_blockhash;
} else {
failures.push((
"recent-blockhash",
format!("Unable to get new blockhash: {}", recent_blockhash),
));
}
if config.monitor_active_stake && current_stake_percent < 80 {
failures.push((
"current-stake",
format!("Current stake is {}%", current_stake_percent),
));
}
if config.validator_identity_pubkeys.is_empty() {
if !vote_accounts.delinquent.is_empty() {
failures.push((
"delinquent",
format!("{} delinquent validators", vote_accounts.delinquent.len()),
))
));
}
} else {
let mut errors = vec![];
for validator_identity in validator_identity_pubkeys.iter() {
for validator_identity in config.validator_identity_pubkeys.iter() {
if vote_accounts
.delinquent
.iter()
@ -201,46 +219,38 @@ fn main() -> Result<(), Box<dyn error::Error>> {
}
}
if errors.is_empty() {
Ok(true)
} else {
Err(io::Error::new(io::ErrorKind::Other, errors.join(",")))
if !errors.is_empty() {
failures.push(("delinquent", errors.join(",")));
}
}
})
.unwrap_or_else(|err| {
notify_msg = format!("solana-watchtower: {}", err.to_string());
failures.into_iter().next() // Only report the first failure if any
}
Err(err) => Some(("rpc", err.to_string())),
};
datapoint_info!("watchtower-sanity", ("ok", failure.is_none(), bool));
if let Some((failure_test_name, failure_error_message)) = &failure {
let notification_msg = format!(
"solana-watchtower: Error: {}: {}",
failure_test_name, failure_error_message
);
if !config.no_duplicate_notifications || last_notification_msg != notification_msg {
notifier.send(&notification_msg);
}
datapoint_error!(
"watchtower-sanity-failure",
("test", "delinquent-validators", String),
("err", err.to_string(), String)
("test", failure_test_name, String),
("err", failure_error_message, String)
);
false
});
datapoint_info!("watchtower-sanity", ("ok", ok, bool));
if !ok {
last_check_notification_sent = true;
if no_duplicate_notifications {
if last_notification_msg != notify_msg {
notifier.send(&notify_msg);
last_notification_msg = notify_msg;
last_notification_msg = notification_msg;
} else {
datapoint_info!(
"watchtower-sanity",
("Suppressing duplicate notification", ok, bool)
);
if !last_notification_msg.is_empty() {
info!("All clear");
notifier.send("solana-watchtower: All clear");
}
} else {
notifier.send(&notify_msg);
last_notification_msg = "".into();
}
} else {
if last_check_notification_sent {
notifier.send("solana-watchtower: All Clear");
}
last_check_notification_sent = false;
last_notification_msg = String::from("");
}
sleep(interval);
sleep(config.interval);
}
}