Restore ledger-tool print and json commands (#5048)

* Restore ledger-tool print and  json commands

* Remove obsolete read_ledger()

core/src/blocktree.rs

@@ -793,70 +793,6 @@ impl Blocktree {
         iter.map(|(_, blob_data)| Blob::new(&blob_data))
     }
 
-    /// Return an iterator for all the entries in the given file.
-    pub fn read_ledger(&self) -> Result<impl Iterator<Item = Entry>> {
-        use crate::entry::EntrySlice;
-        use std::collections::VecDeque;
-
-        struct EntryIterator {
-            db_iterator: Cursor<cf::Data>,
-
-            // TODO: remove me when replay_stage is iterating by block (Blocktree)
-            //    this verification is duplicating that of replay_stage, which
-            //    can do this in parallel
-            blockhash: Option<Hash>,
-            // https://github.com/rust-rocksdb/rust-rocksdb/issues/234
-            //   rocksdb issue: the _blocktree member must be lower in the struct to prevent a crash
-            //   when the db_iterator member above is dropped.
-            //   _blocktree is unused, but dropping _blocktree results in a broken db_iterator
-            //   you have to hold the database open in order to iterate over it, and in order
-            //   for db_iterator to be able to run Drop
-            //    _blocktree: Blocktree,
-            entries: VecDeque<Entry>,
-        }
-
-        impl Iterator for EntryIterator {
-            type Item = Entry;
-
-            fn next(&mut self) -> Option<Entry> {
-                if !self.entries.is_empty() {
-                    return Some(self.entries.pop_front().unwrap());
-                }
-
-                if self.db_iterator.valid() {
-                    if let Some(value) = self.db_iterator.value_bytes() {
-                        if let Ok(next_entries) =
-                            deserialize::<Vec<Entry>>(&value[BLOB_HEADER_SIZE..])
-                        {
-                            if let Some(blockhash) = self.blockhash {
-                                if !next_entries.verify(&blockhash) {
-                                    return None;
-                                }
-                            }
-                            self.db_iterator.next();
-                            if next_entries.is_empty() {
-                                return None;
-                            }
-                            self.entries = VecDeque::from(next_entries);
-                            let entry = self.entries.pop_front().unwrap();
-                            self.blockhash = Some(entry.hash);
-                            return Some(entry);
-                        }
-                    }
-                }
-                None
-            }
-        }
-        let mut db_iterator = self.db.cursor::<cf::Data>()?;
-
-        db_iterator.seek_to_first();
-        Ok(EntryIterator {
-            entries: VecDeque::new(),
-            db_iterator,
-            blockhash: None,
-        })
-    }
-
     pub fn get_slot_entries_with_blob_count(
         &self,
         slot: u64,
@@ -1937,9 +1873,7 @@ pub fn tmp_copy_blocktree(from: &str, name: &str) -> String {
 #[cfg(test)]
 pub mod tests {
     use super::*;
-    use crate::entry::{
+    use crate::entry::{create_ticks, make_tiny_test_entries, Entry, EntrySlice};
-        create_ticks, make_tiny_test_entries, make_tiny_test_entries_from_hash, Entry, EntrySlice,
-    };
     use crate::erasure::{CodingGenerator, ErasureConfig};
     use crate::packet;
     use rand::seq::SliceRandom;
@@ -2467,59 +2401,6 @@ pub mod tests {
         Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
     }
 
-    #[test]
-    pub fn test_genesis_and_entry_iterator() {
-        let entries = make_tiny_test_entries_from_hash(&Hash::default(), 10);
-
-        let ledger_path = get_tmp_ledger_path("test_genesis_and_entry_iterator");
-        {
-            genesis(&ledger_path, &Keypair::new(), &entries).unwrap();
-
-            let ledger = Blocktree::open(&ledger_path).expect("open failed");
-
-            let read_entries: Vec<Entry> =
-                ledger.read_ledger().expect("read_ledger failed").collect();
-            assert!(read_entries.verify(&Hash::default()));
-            assert_eq!(entries, read_entries);
-        }
-
-        Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
-    }
-    #[test]
-    pub fn test_entry_iterator_up_to_consumed() {
-        let entries = make_tiny_test_entries_from_hash(&Hash::default(), 3);
-        let ledger_path = get_tmp_ledger_path("test_genesis_and_entry_iterator");
-        {
-            // put entries except last 2 into ledger
-            genesis(&ledger_path, &Keypair::new(), &entries[..entries.len() - 2]).unwrap();
-
-            let ledger = Blocktree::open(&ledger_path).expect("open failed");
-
-            // now write the last entry, ledger has a hole in it one before the end
-            // +-+-+-+-+-+-+-+    +-+
-            // | | | | | | | |    | |
-            // +-+-+-+-+-+-+-+    +-+
-            ledger
-                .write_entries(
-                    0u64,
-                    0,
-                    (entries.len() - 1) as u64,
-                    16,
-                    &entries[entries.len() - 1..],
-                )
-                .unwrap();
-
-            let read_entries: Vec<Entry> =
-                ledger.read_ledger().expect("read_ledger failed").collect();
-            assert!(read_entries.verify(&Hash::default()));
-
-            // enumeration should stop at the hole
-            assert_eq!(entries[..entries.len() - 2].to_vec(), read_entries);
-        }
-
-        Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
-    }
-
     #[test]
     pub fn test_new_blobs_signal() {
         // Initialize ledger

ledger-tool/src/main.rs

@@ -1,13 +1,66 @@
-use clap::{crate_description, crate_name, crate_version, App, Arg, SubCommand};
+use clap::{crate_description, crate_name, crate_version, value_t, App, Arg, SubCommand};
 use solana::blocktree::Blocktree;
 use solana::blocktree_processor::process_blocktree;
 use solana_sdk::genesis_block::GenesisBlock;
 use std::io::{stdout, Write};
 use std::process::exit;
 
+#[derive(PartialEq)]
+enum LedgerOutputMethod {
+    Print,
+    Json,
+}
+fn output_ledger(blocktree: Blocktree, starting_slot: u64, method: LedgerOutputMethod) {
+    let rooted_slot_iterator = blocktree
+        .rooted_slot_iterator(starting_slot)
+        .unwrap_or_else(|err| {
+            eprintln!(
+                "Failed to load entries starting from slot {}: {:?}",
+                starting_slot, err
+            );
+            exit(1);
+        });
+
+    if method == LedgerOutputMethod::Json {
+        stdout().write_all(b"{\"ledger\":[\n").expect("open array");
+    }
+
+    for (slot, slot_meta) in rooted_slot_iterator {
+        match method {
+            LedgerOutputMethod::Print => println!("Slot {}", slot),
+            LedgerOutputMethod::Json => {
+                serde_json::to_writer(stdout(), &slot_meta).expect("serialize slot_meta");
+                stdout().write_all(b",\n").expect("newline");
+            }
+        }
+
+        let entries = blocktree
+            .get_slot_entries(slot, 0, None)
+            .unwrap_or_else(|err| {
+                eprintln!("Failed to load entries for slot {}: {:?}", slot, err);
+                exit(1);
+            });
+
+        for entry in entries {
+            match method {
+                LedgerOutputMethod::Print => println!("{:?}", entry),
+                LedgerOutputMethod::Json => {
+                    serde_json::to_writer(stdout(), &entry).expect("serialize entry");
+                    stdout().write_all(b",\n").expect("newline");
+                }
+            }
+        }
+    }
+
+    if method == LedgerOutputMethod::Json {
+        stdout().write_all(b"\n]}\n").expect("close array");
+    }
+}
+
 fn main() {
     solana_logger::setup();
-    let matches = App::new(crate_name!()).about(crate_description!())
+    let matches = App::new(crate_name!())
+        .about(crate_description!())
         .version(crate_version!())
         .arg(
             Arg::with_name("ledger")
@@ -19,26 +72,12 @@ fn main() {
                 .help("Use directory for ledger location"),
         )
         .arg(
-            Arg::with_name("head")
+            Arg::with_name("starting_slot")
-                .short("n")
+                .long("starting-slot")
-                .long("head")
                 .value_name("NUM")
                 .takes_value(true)
-                .help("Limit to at most the first NUM entries in ledger\n  (only applies to print and json commands)"),
+                .default_value("0")
-        )
+                .help("Start at this slot (only applies to print and json commands)"),
-        .arg(
-            Arg::with_name("min-hashes")
-                .short("h")
-                .long("min-hashes")
-                .value_name("NUM")
-                .takes_value(true)
-                .help("Skip entries with fewer than NUM hashes\n  (only applies to print and json commands)"),
-        )
-        .arg(
-            Arg::with_name("continue")
-                .short("c")
-                .long("continue")
-                .help("Continue verify even if verification fails"),
         )
         .subcommand(SubCommand::with_name("print").about("Print the ledger"))
         .subcommand(SubCommand::with_name("json").about("Print the ledger in JSON format"))
@@ -63,63 +102,27 @@ fn main() {
         }
     };
 
-    let entries = match blocktree.read_ledger() {
+    let starting_slot = value_t!(matches, "starting_slot", u64).unwrap_or_else(|e| e.exit());
-        Ok(entries) => entries,
-        Err(err) => {
-            eprintln!("Failed to read ledger at {}: {}", ledger_path, err);
-            exit(1);
-        }
-    };
-
-    let head = match matches.value_of("head") {
-        Some(head) => head.parse().expect("please pass a number for --head"),
-        None => <usize>::max_value(),
-    };
-
-    let min_hashes = match matches.value_of("min-hashes") {
-        Some(hashes) => hashes
-            .parse()
-            .expect("please pass a number for --min-hashes"),
-        None => 0,
-    } as u64;
 
     match matches.subcommand() {
         ("print", _) => {
-            for (i, entry) in entries.enumerate() {
+            output_ledger(blocktree, starting_slot, LedgerOutputMethod::Print);
-                if i >= head {
-                    break;
-                }
-
-                if entry.num_hashes < min_hashes {
-                    continue;
-                }
-                println!("{:?}", entry);
-            }
         }
         ("json", _) => {
-            stdout().write_all(b"{\"ledger\":[\n").expect("open array");
+            output_ledger(blocktree, starting_slot, LedgerOutputMethod::Json);
-            for (i, entry) in entries.enumerate() {
-                if i >= head {
-                    break;
-                }
-
-                if entry.num_hashes < min_hashes {
-                    continue;
-                }
-                serde_json::to_writer(stdout(), &entry).expect("serialize");
-                stdout().write_all(b",\n").expect("newline");
-            }
-            stdout().write_all(b"\n]}\n").expect("close array");
         }
-        ("verify", _) => match process_blocktree(&genesis_block, &blocktree, None) {
+        ("verify", _) => {
-            Ok((_bank_forks, bank_forks_info, _)) => {
+            println!("Verifying ledger...");
-                println!("{:?}", bank_forks_info);
+            match process_blocktree(&genesis_block, &blocktree, None) {
+                Ok((_bank_forks, bank_forks_info, _)) => {
+                    println!("{:?}", bank_forks_info);
+                }
+                Err(err) => {
+                    eprintln!("Ledger verification failed: {:?}", err);
+                    exit(1);
+                }
             }
-            Err(err) => {
+        }
-                eprintln!("Ledger verification failed: {:?}", err);
-                exit(1);
-            }
-        },
         ("", _) => {
             eprintln!("{}", matches.usage());
             exit(1);

ledger-tool/tests/basic.rs

@@ -4,7 +4,6 @@ extern crate solana;
 use assert_cmd::prelude::*;
 use solana::blocktree::create_new_tmp_ledger;
 use solana::genesis_utils::create_genesis_block;
-use std::cmp;
 use std::process::Command;
 use std::process::Output;
 
@@ -46,22 +45,5 @@ fn nominal() {
     // Print everything
     let output = run_ledger_tool(&["-l", &ledger_path, "print"]);
     assert!(output.status.success());
-    assert_eq!(count_newlines(&output.stdout), ticks);
+    assert_eq!(count_newlines(&output.stdout), ticks + 1);
-
-    // Only print the first N items
-    let count = cmp::min(genesis_block.ticks_per_slot, 5);
-    let output = run_ledger_tool(&["-l", &ledger_path, "-n", &count.to_string(), "print"]);
-    println!("{:?}", output);
-    assert!(output.status.success());
-    assert_eq!(count_newlines(&output.stdout), count as usize);
-
-    // Skip entries with no hashes
-    let output = run_ledger_tool(&["-l", &ledger_path, "-h", "1", "print"]);
-    assert!(output.status.success());
-    assert_eq!(count_newlines(&output.stdout), ticks);
-
-    // Skip entries with fewer than 2 hashes (skip everything)
-    let output = run_ledger_tool(&["-l", &ledger_path, "-h", "2", "print"]);
-    assert!(output.status.success());
-    assert_eq!(count_newlines(&output.stdout), 0);
 }