From bb5a69aa4dd882530a8d59a883cb572c8efeb550 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 26 Feb 2021 07:37:46 +0000 Subject: [PATCH] ledger-tool cleanup and additions (#15179) (#15554) * Plumb allow-dead-slots to ledger-tool verify * ledger-tool cleanup and add some useful missing args Print root slots and how many unrooted past last root. (cherry picked from commit bbae23358c17276ea2664068746fb6ab92db1a76) Co-authored-by: sakridge --- core/src/replay_stage.rs | 1 + ledger-tool/src/main.rs | 104 +++++++++++++++++++---------- ledger/src/blockstore_processor.rs | 6 +- 3 files changed, 76 insertions(+), 35 deletions(-) diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 98bcdcfc23..a6f0c92324 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -1009,6 +1009,7 @@ impl ReplayStage { Some(replay_vote_sender), None, verify_recyclers, + false, ); let tx_count_after = bank_progress.replay_progress.num_txs; let tx_count = tx_count_after - tx_count_before; diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 5e616a8d68..ad4c3a1756 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -21,7 +21,6 @@ use solana_ledger::{ blockstore::{create_new_ledger, Blockstore, PurgeType}, blockstore_db::{self, AccessType, BlockstoreRecoveryMode, Column, Database}, blockstore_processor::ProcessOptions, - rooted_slot_iterator::RootedSlotIterator, shred::Shred, }; use solana_runtime::{ @@ -211,6 +210,7 @@ fn output_slot( fn output_ledger( blockstore: Blockstore, starting_slot: Slot, + ending_slot: Slot, allow_dead_slots: bool, method: LedgerOutputMethod, num_slots: Option, @@ -231,12 +231,15 @@ fn output_ledger( stdout().write_all(b"{\"ledger\":[\n").expect("open array"); } - let num_slots = num_slots.unwrap_or(std::u64::MAX); + let num_slots = num_slots.unwrap_or(Slot::MAX); let mut num_printed = 0; for (slot, slot_meta) in slot_iterator { if only_rooted && !blockstore.is_root(slot) { continue; } + if slot > ending_slot { + break; + } match method { LedgerOutputMethod::Print => { @@ -865,6 +868,7 @@ fn main() { .about("Print the ledger") .arg(&starting_slot_arg) .arg(&allow_dead_slots_arg) + .arg(&ending_slot_arg) .arg( Arg::with_name("num_slots") .long("num-slots") @@ -884,14 +888,7 @@ fn main() { SubCommand::with_name("copy") .about("Copy the ledger") .arg(&starting_slot_arg) - .arg( - Arg::with_name("ending_slot") - .long("ending-slot") - .value_name("SLOT") - .validator(is_slot) - .takes_value(true) - .help("Slot to stop copy"), - ) + .arg(&ending_slot_arg) .arg( Arg::with_name("target_db") .long("target-db") @@ -1021,6 +1018,7 @@ fn main() { .arg(&hard_forks_arg) .arg(&no_accounts_db_caching_arg) .arg(&bpf_jit_arg) + .arg(&allow_dead_slots_arg) .arg(&max_genesis_archive_unpacked_size_arg) .arg( Arg::with_name("skip_poh_verify") @@ -1293,7 +1291,6 @@ fn main() { .long("max-height") .value_name("NUM") .takes_value(true) - .required(true) .help("Maximum block height") ) .arg( @@ -1350,6 +1347,7 @@ fn main() { ("bigtable", Some(arg_matches)) => bigtable_process_command(&ledger_path, arg_matches), ("print", Some(arg_matches)) => { let starting_slot = value_t_or_exit!(arg_matches, "starting_slot", Slot); + let ending_slot = value_t!(arg_matches, "ending_slot", Slot).unwrap_or(Slot::MAX); let num_slots = value_t!(arg_matches, "num_slots", Slot).ok(); let allow_dead_slots = arg_matches.is_present("allow_dead_slots"); let only_rooted = arg_matches.is_present("only_rooted"); @@ -1361,6 +1359,7 @@ fn main() { wal_recovery_mode, ), starting_slot, + ending_slot, allow_dead_slots, LedgerOutputMethod::Print, num_slots, @@ -1552,6 +1551,7 @@ fn main() { wal_recovery_mode, ), starting_slot, + Slot::MAX, allow_dead_slots, LedgerOutputMethod::Json, None, @@ -1656,6 +1656,7 @@ fn main() { poh_verify: !arg_matches.is_present("skip_poh_verify"), bpf_jit: arg_matches.is_present("bpf_jit"), accounts_db_caching_enabled: !arg_matches.is_present("no_accounts_db_caching"), + allow_dead_slots: arg_matches.is_present("allow_dead_slots"), ..ProcessOptions::default() }; let print_accounts_stats = arg_matches.is_present("print_accounts_stats"); @@ -2695,7 +2696,7 @@ fn main() { let max_height = if let Some(height) = arg_matches.value_of("max_height") { usize::from_str(height).expect("Maximum height must be a number") } else { - panic!("Maximum height must be provided"); + usize::MAX }; let num_roots = if let Some(roots) = arg_matches.value_of("num_roots") { usize::from_str(roots).expect("Number of roots must be a number") @@ -2703,23 +2704,27 @@ fn main() { usize::from_str(DEFAULT_ROOT_COUNT).unwrap() }; - let iter = RootedSlotIterator::new(0, &blockstore).expect("Failed to get rooted slot"); + let iter = blockstore + .rooted_slot_iterator(0) + .expect("Failed to get rooted slot"); - let slot_hash: Vec<_> = iter - .filter_map(|(slot, _meta)| { - if slot <= max_height as u64 { - let blockhash = blockstore - .get_slot_entries(slot, 0) - .unwrap() - .last() - .unwrap() - .hash; - Some((slot, blockhash)) - } else { - None - } - }) - .collect(); + let mut slot_hash = Vec::new(); + for (i, slot) in iter.into_iter().enumerate() { + if i > num_roots { + break; + } + if slot <= max_height as u64 { + let blockhash = blockstore + .get_slot_entries(slot, 0) + .unwrap() + .last() + .unwrap() + .hash; + slot_hash.push((slot, blockhash)); + } else { + break; + } + } let mut output_file: Box = if let Some(path) = arg_matches.value_of("slot_list") { @@ -2744,13 +2749,12 @@ fn main() { }); } ("bounds", Some(arg_matches)) => { - match open_blockstore( + let blockstore = open_blockstore( &ledger_path, AccessType::TryPrimaryThenSecondary, wal_recovery_mode, - ) - .slot_meta_iterator(0) - { + ); + match blockstore.slot_meta_iterator(0) { Ok(metas) => { let all = arg_matches.is_present("all"); @@ -2761,7 +2765,12 @@ fn main() { let first = slots.first().unwrap(); let last = slots.last().unwrap_or(first); if first != last { - println!("Ledger has data for slots {:?} to {:?}", first, last); + println!( + "Ledger has data for {} slots {:?} to {:?}", + slots.len(), + first, + last + ); if all { println!("Non-empty slots: {:?}", slots); } @@ -2769,12 +2778,39 @@ fn main() { println!("Ledger has data for slot {:?}", first); } } + if let Ok(rooted) = blockstore.rooted_slot_iterator(0) { + let mut first_rooted = 0; + let mut last_rooted = 0; + let mut total_rooted = 0; + for (i, slot) in rooted.into_iter().enumerate() { + if i == 0 { + first_rooted = slot; + } + last_rooted = slot; + total_rooted += 1; + } + let mut count_past_root = 0; + for slot in slots.iter().rev() { + if *slot > last_rooted { + count_past_root += 1; + } else { + break; + } + } + println!( + " with {} rooted slots from {:?} to {:?}", + total_rooted, first_rooted, last_rooted + ); + println!(" and {} slots past the last root", count_past_root); + } else { + println!(" with no rooted slots"); + } } Err(err) => { eprintln!("Unable to read the Ledger: {:?}", err); exit(1); } - } + }; } ("analyze-storage", _) => { analyze_storage(&open_database( diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index 06816e2341..5a135745dc 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -367,6 +367,7 @@ pub struct ProcessOptions { pub debug_keys: Option>>, pub account_indexes: HashSet, pub accounts_db_caching_enabled: bool, + pub allow_dead_slots: bool, } pub fn process_blockstore( @@ -596,6 +597,7 @@ fn confirm_full_slot( replay_vote_sender, opts.entry_callback.as_ref(), recyclers, + opts.allow_dead_slots, )?; if !bank.is_complete() { @@ -649,6 +651,7 @@ impl ConfirmationProgress { } } +#[allow(clippy::too_many_arguments)] pub fn confirm_slot( blockstore: &Blockstore, bank: &Arc, @@ -659,13 +662,14 @@ pub fn confirm_slot( replay_vote_sender: Option<&ReplayVoteSender>, entry_callback: Option<&ProcessCallback>, recyclers: &VerifyRecyclers, + allow_dead_slots: bool, ) -> result::Result<(), BlockstoreProcessorError> { let slot = bank.slot(); let (entries, num_shreds, slot_full) = { let mut load_elapsed = Measure::start("load_elapsed"); let load_result = blockstore - .get_slot_entries_with_shred_info(slot, progress.num_shreds, false) + .get_slot_entries_with_shred_info(slot, progress.num_shreds, allow_dead_slots) .map_err(BlockstoreProcessorError::FailedToLoadEntries); load_elapsed.stop(); if load_result.is_err() {