gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Measure heap usage while processing the ledger at validator startup (bp #7667) (#7669)

Browse Source 
automerge
This commit is contained in:
mergify[bot]
2020-01-03 15:42:27 -08:00
committed by Grimes
parent 63db9d6933
commit d6c1cf2499
12 changed files with 60 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
ledger/src/blocktree_processor.rs
View File

@@ -11,6 +11,7 @@ use itertools::Itertools;
use log::*;
use rand::{seq::SliceRandom, thread_rng};
use rayon::{prelude::*, ThreadPool};
use solana_measure::thread_mem_usage;
use solana_metrics::{datapoint, datapoint_error, inc_new_counter_debug};
use solana_rayon_threadlimit::get_thread_count;
use solana_runtime::{
@@ -282,6 +283,9 @@ pub fn process_blocktree_from_root(
opts: &ProcessOptions,
) -> result::Result<(BankForks, Vec<BankForksInfo>, LeaderScheduleCache), BlocktreeProcessorError> {
info!("processing ledger from root slot {}...", bank.slot());
let allocated = thread_mem_usage::Allocatedp::default();
let initial_allocation = allocated.get();
// Starting slot must be a root, and thus has no parents
assert!(bank.parent().is_none());
let start_slot = bank.slot();
@@ -334,8 +338,9 @@ pub fn process_blocktree_from_root(
};
info!(
"ledger processed in {}ms. {} fork{} at {}",
"ledger processed in {}ms. {} MB allocated. {} fork{} at {}",
duration_as_ms(&now.elapsed()),
allocated.since(initial_allocation) / 1_000_000,
bank_forks_info.len(),
if bank_forks_info.len() > 1 { "s" } else { "" },
bank_forks_info
@@ -451,6 +456,9 @@ fn process_next_slots(
// Only process full slots in blocktree_processor, replay_stage
// handles any partials
if next_meta.is_full() {
let allocated = thread_mem_usage::Allocatedp::default();
let initial_allocation = allocated.get();
let next_bank = Arc::new(Bank::new_from_parent(
&bank,
&leader_schedule_cache
@@ -458,7 +466,12 @@ fn process_next_slots(
.unwrap(),
*next_slot,
));
trace!("Add child bank {} of slot={}", next_slot, bank.slot());
trace!(
"New bank for slot {}, parent slot is {}. {} bytes allocated",
next_slot,
bank.slot(),
allocated.since(initial_allocation)
);
pending_slots.push((*next_slot, next_meta, next_bank, bank.last_blockhash()));
} else {
let bfi = BankForksInfo {
@@ -486,6 +499,7 @@ fn process_pending_slots(
let mut fork_info = vec![];
let mut last_status_report = Instant::now();
let mut pending_slots = vec![];
let mut last_root_slot = root_bank.slot();
process_next_slots(
root_bank,
root_meta,
@@ -500,7 +514,10 @@ fn process_pending_slots(
let (slot, meta, bank, last_entry_hash) = pending_slots.pop().unwrap();
if last_status_report.elapsed() > Duration::from_secs(2) {
info!("processing ledger...slot {}", slot);
info!(
"processing ledger: slot={}, last root slot={}",
slot, last_root_slot
);
last_status_report = Instant::now();
}
@@ -509,6 +526,9 @@ fn process_pending_slots(
continue;
}
let allocated = thread_mem_usage::Allocatedp::default();
let initial_allocation = allocated.get();
// Fetch all entries for this slot
let entries = blocktree.get_slot_entries(slot, 0, None).map_err(|err| {
warn!("Failed to load entries for slot {}: {:?}", slot, err);
@@ -531,8 +551,16 @@ fn process_pending_slots(
bank.squash();
pending_slots.clear();
fork_info.clear();
last_root_slot = slot;
}
trace!(
"Bank for {}slot {} is complete. {} bytes allocated",
if last_root_slot == slot { "root " } else { "" },
slot,
allocated.since(initial_allocation)
);
if slot >= dev_halt_at_slot {
let bfi = BankForksInfo { bank_slot: slot };
fork_info.push((bank, bfi));