From 2dfa48daf922844ee32dcac2e41f9c2b15ebca52 Mon Sep 17 00:00:00 2001 From: carllin Date: Tue, 30 Jun 2020 02:20:54 -0700 Subject: [PATCH] Add ability to parse logs in ledger-tool (#10840) Co-authored-by: Carl --- Cargo.lock | 1 + ledger-tool/Cargo.toml | 1 + ledger-tool/src/main.rs | 89 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4834e8e04d..3cbf215a67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4212,6 +4212,7 @@ dependencies = [ "clap", "histogram", "log 0.4.8", + "regex", "serde_json", "serde_yaml", "signal-hook", diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml index b7f4a60288..e29ce3ba28 100644 --- a/ledger-tool/Cargo.toml +++ b/ledger-tool/Cargo.toml @@ -27,6 +27,7 @@ solana-transaction-status = { path = "../transaction-status", version = "1.3.0" solana-version = { path = "../version", version = "1.3.0" } solana-vote-program = { path = "../programs/vote", version = "1.3.0" } tempfile = "3.1.0" +regex = "1" [dev-dependencies] assert_cmd = "1.0" diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 842273e4fb..5194ee3ac6 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -2,10 +2,12 @@ use clap::{ crate_description, crate_name, value_t, value_t_or_exit, values_t_or_exit, App, Arg, ArgMatches, SubCommand, }; +use regex::Regex; use serde_json::json; use solana_clap_utils::input_validators::{is_parsable, is_slot}; use solana_ledger::entry::Entry; use solana_ledger::{ + ancestor_iterator::AncestorIterator, bank_forks_utils, blockstore::Blockstore, blockstore_db::{self, AccessType, Column, Database}, @@ -25,11 +27,11 @@ use solana_sdk::{ }; use solana_vote_program::vote_state::VoteState; use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, BTreeSet, HashMap, HashSet}, convert::TryInto, ffi::OsStr, fs::{self, File}, - io::{self, stdout, Write}, + io::{self, stdout, BufRead, BufReader, Write}, path::{Path, PathBuf}, process::{exit, Command, Stdio}, str::FromStr, @@ -834,6 +836,25 @@ fn main() { .about("Prints the ledger's genesis config") .arg(&max_genesis_archive_unpacked_size_arg) ) + .subcommand( + SubCommand::with_name("parse_full_frozen") + .about("Parses log for information about critical events about ancestors of the given `ending_slot`") + .arg(&starting_slot_arg) + .arg( + Arg::with_name("ending_slot") + .long("ending-slot") + .value_name("SLOT") + .takes_value(true) + .help("The last slot to iterate to"), + ) + .arg( + Arg::with_name("log_path") + .long("log-path") + .value_name("PATH") + .takes_value(true) + .help("path to log file to parse"), + ) + ) .subcommand( SubCommand::with_name("genesis-hash") .about("Prints the ledger's genesis hash") @@ -1134,6 +1155,70 @@ fn main() { } } } + ("parse_full_frozen", Some(arg_matches)) => { + let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); + let ending_slot = value_t_or_exit!(arg_matches, "ending_slot", Slot); + let blockstore = open_blockstore(&ledger_path, AccessType::TryPrimaryThenSecondary); + let mut ancestors = BTreeSet::new(); + if blockstore.meta(ending_slot).unwrap().is_none() { + panic!("Ending slot doesn't exist"); + } + for a in AncestorIterator::new(ending_slot, &blockstore) { + ancestors.insert(a); + if a <= starting_slot { + break; + } + } + println!("ancestors: {:?}", ancestors.iter()); + + let mut frozen = BTreeMap::new(); + let mut full = BTreeMap::new(); + let frozen_regex = Regex::new(r"bank frozen: (\d*)").unwrap(); + let full_regex = Regex::new(r"slot (\d*) is full").unwrap(); + + let log_file = PathBuf::from(value_t_or_exit!(arg_matches, "log_path", String)); + let f = BufReader::new(File::open(log_file).unwrap()); + println!("Reading log file"); + for line in f.lines() { + if let Ok(line) = line { + let parse_results = { + if let Some(slot_string) = frozen_regex.captures_iter(&line).next() { + Some((slot_string, &mut frozen)) + } else if let Some(slot_string) = full_regex.captures_iter(&line).next() { + Some((slot_string, &mut full)) + } else { + None + } + }; + + if let Some((slot_string, map)) = parse_results { + let slot = slot_string + .get(1) + .expect("Only one match group") + .as_str() + .parse::() + .unwrap(); + if ancestors.contains(&slot) && !map.contains_key(&slot) { + map.insert(slot, line); + } + if slot == starting_slot + && frozen.contains_key(&slot) + && full.contains_key(&slot) + { + break; + } + } + } + } + + for ((slot1, frozen_log), (slot2, full_log)) in frozen.iter().zip(full.iter()) { + assert_eq!(slot1, slot2); + println!( + "Slot: {}\n, full: {}\n, frozen: {}", + slot1, full_log, frozen_log + ); + } + } ("verify", Some(arg_matches)) => { let process_options = ProcessOptions { dev_halt_at_slot: value_t!(arg_matches, "halt_at_slot", Slot).ok(),