gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add --print-account-stats/--snapshot-archive-path arguments to ledger-tool (#10945)

Browse Source 
Allows for seeing how the accounts are being stored
and specifying a different snapshot source directory.

(cherry picked from commit 6429042b6e)
This commit is contained in:
sakridge
2020-07-08 09:32:11 -07:00
committed by Trent Nelson
parent 159722a960
commit aebb8187a8
3 changed files with 104 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
ledger-tool/src/main.rs
View File

@ -572,6 +572,7 @@ fn load_bank_forks(
process_options: ProcessOptions,
access_type: AccessType,
wal_recovery_mode: Option<BlockstoreRecoveryMode>,
snapshot_archive_path: Option<PathBuf>,
) -> bank_forks_utils::LoadResult {
let blockstore = open_blockstore(&ledger_path, access_type, wal_recovery_mode);
let snapshot_path = ledger_path.clone().join(if blockstore.is_primary_access() {
@ -582,9 +583,11 @@ fn load_bank_forks(
let snapshot_config = if arg_matches.is_present("no_snapshot") {
None
} else {
let snapshot_package_output_path =
snapshot_archive_path.unwrap_or_else(|| ledger_path.clone());
Some(SnapshotConfig {
snapshot_interval_slots: 0, // Value doesn't matter
snapshot_package_output_path: ledger_path.clone(),
snapshot_package_output_path,
snapshot_path,
compression: CompressionType::Bzip2,
snapshot_version: SnapshotVersion::default(),
@ -592,7 +595,7 @@ fn load_bank_forks(
};
let account_paths = if let Some(account_paths) = arg_matches.value_of("account_paths") {
if !blockstore.is_primary_access() {
// Be defenstive, when default account dir is explicitly specified, it's still possible
// Be defensive, when default account dir is explicitly specified, it's still possible
// to wipe the dir possibly shared by the running validator!
eprintln!("Error: custom accounts path is not supported under secondary access");
exit(1);
@ -727,6 +730,14 @@ fn main() {
"Mode to recovery the ledger db write ahead log."
),
)
.arg(
Arg::with_name("snapshot_archive_path")
.long("snapshot-archive-path")
.value_name("DIR")
.takes_value(true)
.global(true)
.help("Use DIR for ledger location"),
)
.subcommand(
SubCommand::with_name("print")
.about("Print the ledger")
@ -827,6 +838,12 @@ fn main() {
.takes_value(false)
.help("Skip ledger PoH verification"),
)
.arg(
Arg::with_name("print_accounts_stats")
.long("print-accounts-stats")
.takes_value(false)
.help("After verifying the ledger, print some information about the account stores."),
)
).subcommand(
SubCommand::with_name("graph")
.about("Create a Graphviz rendering of the ledger")
@ -1002,6 +1019,10 @@ fn main() {
exit(1);
});
let snapshot_archive_path = value_t!(matches, "snapshot_archive_path", String)
.ok()
.map(PathBuf::from);
let wal_recovery_mode = matches
.value_of("wal_recovery_mode")
.map(BlockstoreRecoveryMode::from);
@ -1046,6 +1067,7 @@ fn main() {
process_options,
AccessType::TryPrimaryThenSecondary,
wal_recovery_mode,
snapshot_archive_path,
) {
Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
println!(
@ -1182,23 +1204,29 @@ fn main() {
poh_verify: !arg_matches.is_present("skip_poh_verify"),
..ProcessOptions::default()
};
let print_accounts_stats = arg_matches.is_present("print_accounts_stats");
println!(
"genesis hash: {}",
open_genesis_config_by(&ledger_path, arg_matches).hash()
);
load_bank_forks(
let (bank_forks, _, _) = load_bank_forks(
arg_matches,
&ledger_path,
&open_genesis_config_by(&ledger_path, arg_matches),
process_options,
AccessType::TryPrimaryThenSecondary,
wal_recovery_mode,
snapshot_archive_path,
)
.unwrap_or_else(|err| {
eprintln!("Ledger verification failed: {:?}", err);
exit(1);
});
if print_accounts_stats {
let working_bank = bank_forks.working_bank();
working_bank.print_accounts_stats();
}
println!("Ok");
}
("graph", Some(arg_matches)) => {
@ -1218,6 +1246,7 @@ fn main() {
process_options,
AccessType::TryPrimaryThenSecondary,
wal_recovery_mode,
snapshot_archive_path,
) {
Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
let dot = graph_forks(&bank_forks, arg_matches.is_present("include_all_votes"));
@ -1270,6 +1299,7 @@ fn main() {
process_options,
AccessType::TryPrimaryThenSecondary,
wal_recovery_mode,
snapshot_archive_path,
) {
Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
let bank = bank_forks
@ -1366,6 +1396,7 @@ fn main() {
process_options,
AccessType::TryPrimaryThenSecondary,
wal_recovery_mode,
snapshot_archive_path,
) {
Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
let slot = bank_forks.working_bank().slot();
@ -1420,6 +1451,7 @@ fn main() {
process_options,
AccessType::TryPrimaryThenSecondary,
wal_recovery_mode,
snapshot_archive_path,
) {
Ok((bank_forks, _leader_schedule_cache, _snapshot_hash)) => {
let slot = bank_forks.working_bank().slot();

127
runtime/src/accounts_db.rs
View File

@ -2122,6 +2122,53 @@ impl AccountsDB {
}
}
}
pub fn print_accounts_stats(&self, label: &'static str) {
self.print_index(label);
self.print_count_and_status(label);
}
fn print_index(&self, label: &'static str) {
let mut roots: Vec<_> = self
.accounts_index
.read()
.unwrap()
.roots
.iter()
.cloned()
.collect();
roots.sort();
info!("{}: accounts_index roots: {:?}", label, roots,);
for (pubkey, list) in &self.accounts_index.read().unwrap().account_maps {
info!(" key: {}", pubkey);
info!(" slots: {:?}", *list.1.read().unwrap());
}
}
fn print_count_and_status(&self, label: &'static str) {
let storage = self.storage.read().unwrap();
let mut slots: Vec<_> = storage.0.keys().cloned().collect();
slots.sort();
info!("{}: count_and status for {} slots:", label, slots.len());
for slot in &slots {
let slot_stores = storage.0.get(slot).unwrap();
let mut ids: Vec<_> = slot_stores.keys().cloned().collect();
ids.sort();
for id in &ids {
let entry = slot_stores.get(id).unwrap();
info!(
" slot: {} id: {} count_and_status: {:?} approx_store_count: {} len: {} capacity: {}",
slot,
id,
*entry.count_and_status.read().unwrap(),
entry.approx_store_count.load(Ordering::Relaxed),
entry.accounts.len(),
entry.accounts.capacity(),
);
}
}
}
}
#[cfg(test)]
@ -2704,7 +2751,7 @@ pub mod tests {
accounts.store(1, &[(&pubkey, &account)]);
//slot is gone
print_accounts("pre-clean", &accounts);
accounts.print_accounts_stats("pre-clean");
accounts.clean_accounts();
accounts.process_dead_slots(None);
assert!(accounts.storage.read().unwrap().0.get(&0).is_none());
@ -2884,50 +2931,6 @@ pub mod tests {
assert_eq!(accounts.uncleaned_root_count(), 0);
}
fn print_accounts(label: &'static str, accounts: &AccountsDB) {
print_index(label, accounts);
print_count_and_status(label, accounts);
}
fn print_index(label: &'static str, accounts: &AccountsDB) {
let mut roots: Vec<_> = accounts
.accounts_index
.read()
.unwrap()
.roots
.iter()
.cloned()
.collect();
roots.sort();
info!("{}: accounts.accounts_index roots: {:?}", label, roots,);
for (pubkey, list) in &accounts.accounts_index.read().unwrap().account_maps {
info!(" key: {}", pubkey);
info!(" slots: {:?}", *list.1.read().unwrap());
}
}
fn print_count_and_status(label: &'static str, accounts: &AccountsDB) {
let storage = accounts.storage.read().unwrap();
let mut slots: Vec<_> = storage.0.keys().cloned().collect();
slots.sort();
info!("{}: count_and status for {} slots:", label, slots.len());
for slot in &slots {
let slot_stores = storage.0.get(slot).unwrap();
let mut ids: Vec<_> = slot_stores.keys().cloned().collect();
ids.sort();
for id in &ids {
let entry = slot_stores.get(id).unwrap();
info!(
" slot: {} id: {} count_and_status: {:?}",
slot,
id,
*entry.count_and_status.read().unwrap()
);
}
}
}
#[test]
fn test_accounts_db_serialize1() {
solana_logger::setup();
@ -2996,7 +2999,7 @@ pub mod tests {
accounts.bank_hashes.read().unwrap().get(&latest_slot)
);
print_count_and_status("daccounts", &daccounts);
daccounts.print_count_and_status("daccounts");
// Don't check the first 35 accounts which have not been modified on slot 0
check_accounts(&daccounts, &pubkeys[35..], 0, 65, 37);
@ -3031,7 +3034,7 @@ pub mod tests {
fn reconstruct_accounts_db_via_serialization(accounts: &AccountsDB, slot: Slot) -> AccountsDB {
let daccounts =
crate::serde_snapshot::reconstruct_accounts_db_via_serialization(accounts, slot);
print_count_and_status("daccounts", &daccounts);
daccounts.print_count_and_status("daccounts");
daccounts
}
@ -3077,11 +3080,11 @@ pub mod tests {
current_slot += 1;
accounts.add_root(current_slot);
print_accounts("pre_purge", &accounts);
accounts.print_accounts_stats("pre_purge");
accounts.clean_accounts();
print_accounts("post_purge", &accounts);
accounts.print_accounts_stats("post_purge");
// Make sure the index is not touched
assert_eq!(
@ -3137,7 +3140,7 @@ pub mod tests {
accounts.set_hash(current_slot, current_slot - 1);
accounts.add_root(current_slot);
print_accounts("pre_purge", &accounts);
accounts.print_accounts_stats("pre_purge");
let ancestors = linear_ancestors(current_slot);
info!("ancestors: {:?}", ancestors);
@ -3151,7 +3154,7 @@ pub mod tests {
hash
);
print_accounts("post_purge", &accounts);
accounts.print_accounts_stats("post_purge");
// Make sure the index is for pubkey cleared
assert!(accounts
@ -3213,14 +3216,14 @@ pub mod tests {
assert_load_account(&accounts, current_slot, pubkey, zero_lamport);
print_accounts("accounts", &accounts);
accounts.print_accounts_stats("accounts");
accounts.clean_accounts();
print_accounts("accounts_post_purge", &accounts);
accounts.print_accounts_stats("accounts_post_purge");
let accounts = reconstruct_accounts_db_via_serialization(&accounts, current_slot);
print_accounts("reconstructed", &accounts);
accounts.print_accounts_stats("reconstructed");
assert_load_account(&accounts, current_slot, pubkey, zero_lamport);
}
@ -3267,12 +3270,12 @@ pub mod tests {
accounts.store(current_slot, &[(&dummy_pubkey, &dummy_account)]);
accounts.add_root(current_slot);
print_accounts("pre_f", &accounts);
accounts.print_accounts_stats("pre_f");
accounts.update_accounts_hash(4, &HashMap::default());
let accounts = f(accounts, current_slot);
print_accounts("post_f", &accounts);
accounts.print_accounts_stats("post_f");
assert_load_account(&accounts, current_slot, pubkey, some_lamport);
assert_load_account(&accounts, current_slot, purged_pubkey1, 0);
@ -3296,7 +3299,7 @@ pub mod tests {
solana_logger::setup();
with_chained_zero_lamport_accounts(|accounts, current_slot| {
let accounts = reconstruct_accounts_db_via_serialization(&accounts, current_slot);
print_accounts("after_reconstruct", &accounts);
accounts.print_accounts_stats("after_reconstruct");
accounts.clean_accounts();
reconstruct_accounts_db_via_serialization(&accounts, current_slot)
});
@ -3394,7 +3397,7 @@ pub mod tests {
let account1 = Account::new(2, 0, &key);
db.store(1, &[(&key1, &account1)]);
print_accounts("pre", &db);
db.print_accounts_stats("pre");
let slots: HashSet<Slot> = HashSet::from_iter(vec![1].into_iter());
let purge_keys = vec![(key1, slots)];
@ -3405,7 +3408,7 @@ pub mod tests {
db.handle_dead_keys(dead_keys);
print_accounts("post", &db);
db.print_accounts_stats("post");
let ancestors = vec![(2, 0)].into_iter().collect();
assert_eq!(db.load_slow(&ancestors, &key1).unwrap().0.lamports, 3);
}
@ -3959,11 +3962,11 @@ pub mod tests {
accounts.store(current_slot, &[(&dummy_pubkey, &dummy_account)]);
accounts.add_root(current_slot);
print_count_and_status("before reconstruct", &accounts);
accounts.print_count_and_status("before reconstruct");
let accounts = reconstruct_accounts_db_via_serialization(&accounts, current_slot);
print_count_and_status("before purge zero", &accounts);
accounts.print_count_and_status("before purge zero");
accounts.clean_accounts();
print_count_and_status("after purge zero", &accounts);
accounts.print_count_and_status("after purge zero");
assert_load_account(&accounts, current_slot, pubkey, old_lamport);
assert_load_account(&accounts, current_slot, purged_pubkey1, 0);

4
runtime/src/bank.rs
View File

@ -2890,6 +2890,10 @@ impl Bank {
self.rc.accounts.accounts_db.shrink_all_slots();
}
pub fn print_accounts_stats(&self) {
self.rc.accounts.accounts_db.print_accounts_stats("");
}
pub fn process_stale_slot_with_budget(
&self,
mut consumed_budget: usize,