diff --git a/core/src/blocktree.rs b/core/src/blocktree.rs index 5a0ce818fd..027cce3732 100644 --- a/core/src/blocktree.rs +++ b/core/src/blocktree.rs @@ -793,70 +793,6 @@ impl Blocktree { iter.map(|(_, blob_data)| Blob::new(&blob_data)) } - /// Return an iterator for all the entries in the given file. - pub fn read_ledger(&self) -> Result> { - use crate::entry::EntrySlice; - use std::collections::VecDeque; - - struct EntryIterator { - db_iterator: Cursor, - - // TODO: remove me when replay_stage is iterating by block (Blocktree) - // this verification is duplicating that of replay_stage, which - // can do this in parallel - blockhash: Option, - // https://github.com/rust-rocksdb/rust-rocksdb/issues/234 - // rocksdb issue: the _blocktree member must be lower in the struct to prevent a crash - // when the db_iterator member above is dropped. - // _blocktree is unused, but dropping _blocktree results in a broken db_iterator - // you have to hold the database open in order to iterate over it, and in order - // for db_iterator to be able to run Drop - // _blocktree: Blocktree, - entries: VecDeque, - } - - impl Iterator for EntryIterator { - type Item = Entry; - - fn next(&mut self) -> Option { - if !self.entries.is_empty() { - return Some(self.entries.pop_front().unwrap()); - } - - if self.db_iterator.valid() { - if let Some(value) = self.db_iterator.value_bytes() { - if let Ok(next_entries) = - deserialize::>(&value[BLOB_HEADER_SIZE..]) - { - if let Some(blockhash) = self.blockhash { - if !next_entries.verify(&blockhash) { - return None; - } - } - self.db_iterator.next(); - if next_entries.is_empty() { - return None; - } - self.entries = VecDeque::from(next_entries); - let entry = self.entries.pop_front().unwrap(); - self.blockhash = Some(entry.hash); - return Some(entry); - } - } - } - None - } - } - let mut db_iterator = self.db.cursor::()?; - - db_iterator.seek_to_first(); - Ok(EntryIterator { - entries: VecDeque::new(), - db_iterator, - blockhash: None, - }) - } - pub fn get_slot_entries_with_blob_count( &self, slot: u64, @@ -1937,9 +1873,7 @@ pub fn tmp_copy_blocktree(from: &str, name: &str) -> String { #[cfg(test)] pub mod tests { use super::*; - use crate::entry::{ - create_ticks, make_tiny_test_entries, make_tiny_test_entries_from_hash, Entry, EntrySlice, - }; + use crate::entry::{create_ticks, make_tiny_test_entries, Entry, EntrySlice}; use crate::erasure::{CodingGenerator, ErasureConfig}; use crate::packet; use rand::seq::SliceRandom; @@ -2467,59 +2401,6 @@ pub mod tests { Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction"); } - #[test] - pub fn test_genesis_and_entry_iterator() { - let entries = make_tiny_test_entries_from_hash(&Hash::default(), 10); - - let ledger_path = get_tmp_ledger_path("test_genesis_and_entry_iterator"); - { - genesis(&ledger_path, &Keypair::new(), &entries).unwrap(); - - let ledger = Blocktree::open(&ledger_path).expect("open failed"); - - let read_entries: Vec = - ledger.read_ledger().expect("read_ledger failed").collect(); - assert!(read_entries.verify(&Hash::default())); - assert_eq!(entries, read_entries); - } - - Blocktree::destroy(&ledger_path).expect("Expected successful database destruction"); - } - #[test] - pub fn test_entry_iterator_up_to_consumed() { - let entries = make_tiny_test_entries_from_hash(&Hash::default(), 3); - let ledger_path = get_tmp_ledger_path("test_genesis_and_entry_iterator"); - { - // put entries except last 2 into ledger - genesis(&ledger_path, &Keypair::new(), &entries[..entries.len() - 2]).unwrap(); - - let ledger = Blocktree::open(&ledger_path).expect("open failed"); - - // now write the last entry, ledger has a hole in it one before the end - // +-+-+-+-+-+-+-+ +-+ - // | | | | | | | | | | - // +-+-+-+-+-+-+-+ +-+ - ledger - .write_entries( - 0u64, - 0, - (entries.len() - 1) as u64, - 16, - &entries[entries.len() - 1..], - ) - .unwrap(); - - let read_entries: Vec = - ledger.read_ledger().expect("read_ledger failed").collect(); - assert!(read_entries.verify(&Hash::default())); - - // enumeration should stop at the hole - assert_eq!(entries[..entries.len() - 2].to_vec(), read_entries); - } - - Blocktree::destroy(&ledger_path).expect("Expected successful database destruction"); - } - #[test] pub fn test_new_blobs_signal() { // Initialize ledger diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 216bc76d41..1694d6b0a5 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1,13 +1,66 @@ -use clap::{crate_description, crate_name, crate_version, App, Arg, SubCommand}; +use clap::{crate_description, crate_name, crate_version, value_t, App, Arg, SubCommand}; use solana::blocktree::Blocktree; use solana::blocktree_processor::process_blocktree; use solana_sdk::genesis_block::GenesisBlock; use std::io::{stdout, Write}; use std::process::exit; +#[derive(PartialEq)] +enum LedgerOutputMethod { + Print, + Json, +} +fn output_ledger(blocktree: Blocktree, starting_slot: u64, method: LedgerOutputMethod) { + let rooted_slot_iterator = blocktree + .rooted_slot_iterator(starting_slot) + .unwrap_or_else(|err| { + eprintln!( + "Failed to load entries starting from slot {}: {:?}", + starting_slot, err + ); + exit(1); + }); + + if method == LedgerOutputMethod::Json { + stdout().write_all(b"{\"ledger\":[\n").expect("open array"); + } + + for (slot, slot_meta) in rooted_slot_iterator { + match method { + LedgerOutputMethod::Print => println!("Slot {}", slot), + LedgerOutputMethod::Json => { + serde_json::to_writer(stdout(), &slot_meta).expect("serialize slot_meta"); + stdout().write_all(b",\n").expect("newline"); + } + } + + let entries = blocktree + .get_slot_entries(slot, 0, None) + .unwrap_or_else(|err| { + eprintln!("Failed to load entries for slot {}: {:?}", slot, err); + exit(1); + }); + + for entry in entries { + match method { + LedgerOutputMethod::Print => println!("{:?}", entry), + LedgerOutputMethod::Json => { + serde_json::to_writer(stdout(), &entry).expect("serialize entry"); + stdout().write_all(b",\n").expect("newline"); + } + } + } + } + + if method == LedgerOutputMethod::Json { + stdout().write_all(b"\n]}\n").expect("close array"); + } +} + fn main() { solana_logger::setup(); - let matches = App::new(crate_name!()).about(crate_description!()) + let matches = App::new(crate_name!()) + .about(crate_description!()) .version(crate_version!()) .arg( Arg::with_name("ledger") @@ -19,26 +72,12 @@ fn main() { .help("Use directory for ledger location"), ) .arg( - Arg::with_name("head") - .short("n") - .long("head") + Arg::with_name("starting_slot") + .long("starting-slot") .value_name("NUM") .takes_value(true) - .help("Limit to at most the first NUM entries in ledger\n (only applies to print and json commands)"), - ) - .arg( - Arg::with_name("min-hashes") - .short("h") - .long("min-hashes") - .value_name("NUM") - .takes_value(true) - .help("Skip entries with fewer than NUM hashes\n (only applies to print and json commands)"), - ) - .arg( - Arg::with_name("continue") - .short("c") - .long("continue") - .help("Continue verify even if verification fails"), + .default_value("0") + .help("Start at this slot (only applies to print and json commands)"), ) .subcommand(SubCommand::with_name("print").about("Print the ledger")) .subcommand(SubCommand::with_name("json").about("Print the ledger in JSON format")) @@ -63,63 +102,27 @@ fn main() { } }; - let entries = match blocktree.read_ledger() { - Ok(entries) => entries, - Err(err) => { - eprintln!("Failed to read ledger at {}: {}", ledger_path, err); - exit(1); - } - }; - - let head = match matches.value_of("head") { - Some(head) => head.parse().expect("please pass a number for --head"), - None => ::max_value(), - }; - - let min_hashes = match matches.value_of("min-hashes") { - Some(hashes) => hashes - .parse() - .expect("please pass a number for --min-hashes"), - None => 0, - } as u64; + let starting_slot = value_t!(matches, "starting_slot", u64).unwrap_or_else(|e| e.exit()); match matches.subcommand() { ("print", _) => { - for (i, entry) in entries.enumerate() { - if i >= head { - break; - } - - if entry.num_hashes < min_hashes { - continue; - } - println!("{:?}", entry); - } + output_ledger(blocktree, starting_slot, LedgerOutputMethod::Print); } ("json", _) => { - stdout().write_all(b"{\"ledger\":[\n").expect("open array"); - for (i, entry) in entries.enumerate() { - if i >= head { - break; - } - - if entry.num_hashes < min_hashes { - continue; - } - serde_json::to_writer(stdout(), &entry).expect("serialize"); - stdout().write_all(b",\n").expect("newline"); - } - stdout().write_all(b"\n]}\n").expect("close array"); + output_ledger(blocktree, starting_slot, LedgerOutputMethod::Json); } - ("verify", _) => match process_blocktree(&genesis_block, &blocktree, None) { - Ok((_bank_forks, bank_forks_info, _)) => { - println!("{:?}", bank_forks_info); + ("verify", _) => { + println!("Verifying ledger..."); + match process_blocktree(&genesis_block, &blocktree, None) { + Ok((_bank_forks, bank_forks_info, _)) => { + println!("{:?}", bank_forks_info); + } + Err(err) => { + eprintln!("Ledger verification failed: {:?}", err); + exit(1); + } } - Err(err) => { - eprintln!("Ledger verification failed: {:?}", err); - exit(1); - } - }, + } ("", _) => { eprintln!("{}", matches.usage()); exit(1); diff --git a/ledger-tool/tests/basic.rs b/ledger-tool/tests/basic.rs index d3b631b00b..48e0a5b7f5 100644 --- a/ledger-tool/tests/basic.rs +++ b/ledger-tool/tests/basic.rs @@ -4,7 +4,6 @@ extern crate solana; use assert_cmd::prelude::*; use solana::blocktree::create_new_tmp_ledger; use solana::genesis_utils::create_genesis_block; -use std::cmp; use std::process::Command; use std::process::Output; @@ -46,22 +45,5 @@ fn nominal() { // Print everything let output = run_ledger_tool(&["-l", &ledger_path, "print"]); assert!(output.status.success()); - assert_eq!(count_newlines(&output.stdout), ticks); - - // Only print the first N items - let count = cmp::min(genesis_block.ticks_per_slot, 5); - let output = run_ledger_tool(&["-l", &ledger_path, "-n", &count.to_string(), "print"]); - println!("{:?}", output); - assert!(output.status.success()); - assert_eq!(count_newlines(&output.stdout), count as usize); - - // Skip entries with no hashes - let output = run_ledger_tool(&["-l", &ledger_path, "-h", "1", "print"]); - assert!(output.status.success()); - assert_eq!(count_newlines(&output.stdout), ticks); - - // Skip entries with fewer than 2 hashes (skip everything) - let output = run_ledger_tool(&["-l", &ledger_path, "-h", "2", "print"]); - assert!(output.status.success()); - assert_eq!(count_newlines(&output.stdout), 0); + assert_eq!(count_newlines(&output.stdout), ticks + 1); }