gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Don't load all accounts into memory for capitalization check (#14957)

Browse Source 
* Don't load all accounts into memory for capitalization check
This commit is contained in:
carllin
2021-02-03 15:00:42 -08:00
committed by GitHub
parent 8e93a784f3
commit 2970b59853
3 changed files with 143 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
runtime/src/accounts_index.rs
View File

@@ -5,6 +5,7 @@ use crate::{
};
use dashmap::DashSet;
use ouroboros::self_referencing;
use solana_measure::measure::Measure;
use solana_sdk::{
clock::Slot,
pubkey::{Pubkey, PUBKEY_BYTES},
@@ -260,6 +261,7 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
fn do_checked_scan_accounts<F, R>(
&self,
metric_name: &'static str,
ancestors: &Ancestors,
func: F,
scan_type: ScanTypes<R>,
@@ -402,7 +404,8 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
*/
match scan_type {
ScanTypes::Unindexed(range) => {
self.do_scan_accounts(ancestors, func, range, Some(max_root));
// Pass "" not to log metrics, so RPC doesn't get spammy
self.do_scan_accounts(metric_name, ancestors, func, range, Some(max_root));
}
ScanTypes::Indexed(IndexKey::ProgramId(program_id)) => {
self.do_scan_secondary_index(
@@ -443,12 +446,17 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
}
}
fn do_unchecked_scan_accounts<F, R>(&self, ancestors: &Ancestors, func: F, range: Option<R>)
where
fn do_unchecked_scan_accounts<F, R>(
&self,
metric_name: &'static str,
ancestors: &Ancestors,
func: F,
range: Option<R>,
) where
F: FnMut(&Pubkey, (&T, Slot)),
R: RangeBounds<Pubkey>,
{
self.do_scan_accounts(ancestors, func, range, None);
self.do_scan_accounts(metric_name, ancestors, func, range, None);
}
// Scan accounts and return latest version of each account that is either:
@@ -456,6 +464,7 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
// 2) present in ancestors
fn do_scan_accounts<F, R>(
&self,
metric_name: &'static str,
ancestors: &Ancestors,
mut func: F,
range: Option<R>,
@@ -466,13 +475,46 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
{
// TODO: expand to use mint index to find the `pubkey_list` below more efficiently
// instead of scanning the entire range
let mut total_elapsed_timer = Measure::start("total");
let mut num_keys_iterated = 0;
let mut latest_slot_elapsed = 0;
let mut load_account_elapsed = 0;
let mut read_lock_elapsed = 0;
let mut iterator_elapsed = 0;
let mut iterator_timer = Measure::start("iterator_elapsed");
for pubkey_list in self.iter(range) {
iterator_timer.stop();
iterator_elapsed += iterator_timer.as_us();
for (pubkey, list) in pubkey_list {
num_keys_iterated += 1;
let mut read_lock_timer = Measure::start("read_lock");
let list_r = &list.slot_list.read().unwrap();
read_lock_timer.stop();
read_lock_elapsed += read_lock_timer.as_us();
let mut latest_slot_timer = Measure::start("latest_slot");
if let Some(index) = self.latest_slot(Some(ancestors), &list_r, max_root) {
latest_slot_timer.stop();
latest_slot_elapsed += latest_slot_timer.as_us();
let mut load_account_timer = Measure::start("load_account");
func(&pubkey, (&list_r[index].1, list_r[index].0));
load_account_timer.stop();
load_account_elapsed += load_account_timer.as_us();
}
}
iterator_timer = Measure::start("iterator_elapsed");
}
total_elapsed_timer.stop();
if !metric_name.is_empty() {
datapoint_info!(
metric_name,
("total_elapsed", total_elapsed_timer.as_us(), i64),
("latest_slot_elapsed", latest_slot_elapsed, i64),
("read_lock_elapsed", read_lock_elapsed, i64),
("load_account_elapsed", load_account_elapsed, i64),
("iterator_elapsed", iterator_elapsed, i64),
("num_keys_iterated", num_keys_iterated, i64),
)
}
}
@@ -577,24 +619,39 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
where
F: FnMut(&Pubkey, (&T, Slot)),
{
self.do_checked_scan_accounts(ancestors, func, ScanTypes::Unindexed(None::<Range<Pubkey>>));
// Pass "" not to log metrics, so RPC doesn't get spammy
self.do_checked_scan_accounts(
"",
ancestors,
func,
ScanTypes::Unindexed(None::<Range<Pubkey>>),
);
}
pub(crate) fn unchecked_scan_accounts<F>(&self, ancestors: &Ancestors, func: F)
where
pub(crate) fn unchecked_scan_accounts<F>(
&self,
metric_name: &'static str,
ancestors: &Ancestors,
func: F,
) where
F: FnMut(&Pubkey, (&T, Slot)),
{
self.do_unchecked_scan_accounts(ancestors, func, None::<Range<Pubkey>>);
self.do_unchecked_scan_accounts(metric_name, ancestors, func, None::<Range<Pubkey>>);
}
/// call func with every pubkey and index visible from a given set of ancestors with range
pub(crate) fn range_scan_accounts<F, R>(&self, ancestors: &Ancestors, range: R, func: F)
where
pub(crate) fn range_scan_accounts<F, R>(
&self,
metric_name: &'static str,
ancestors: &Ancestors,
range: R,
func: F,
) where
F: FnMut(&Pubkey, (&T, Slot)),
R: RangeBounds<Pubkey>,
{
// Only the rent logic should be calling this, which doesn't need the safety checks
self.do_unchecked_scan_accounts(ancestors, func, Some(range));
self.do_unchecked_scan_accounts(metric_name, ancestors, func, Some(range));
}
/// call func with every pubkey and index visible from a given set of ancestors
@@ -602,7 +659,9 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
where
F: FnMut(&Pubkey, (&T, Slot)),
{
// Pass "" not to log metrics, so RPC doesn't get spammy
self.do_checked_scan_accounts(
"",
ancestors,
func,
ScanTypes::<Range<Pubkey>>::Indexed(index_key),
@@ -1128,7 +1187,7 @@ pub mod tests {
assert!(index.get(&key.pubkey(), None, None).is_none());
let mut num = 0;
index.unchecked_scan_accounts(&ancestors, |_pubkey, _index| num += 1);
index.unchecked_scan_accounts("", &ancestors, |_pubkey, _index| num += 1);
assert_eq!(num, 0);
}
@@ -1153,7 +1212,7 @@ pub mod tests {
assert!(index.get(&key.pubkey(), None, None).is_none());
let mut num = 0;
index.unchecked_scan_accounts(&ancestors, |_pubkey, _index| num += 1);
index.unchecked_scan_accounts("", &ancestors, |_pubkey, _index| num += 1);
assert_eq!(num, 0);
}
@@ -1177,7 +1236,7 @@ pub mod tests {
assert!(index.get(&key.pubkey(), Some(&ancestors), None).is_none());
let mut num = 0;
index.unchecked_scan_accounts(&ancestors, |_pubkey, _index| num += 1);
index.unchecked_scan_accounts("", &ancestors, |_pubkey, _index| num += 1);
assert_eq!(num, 0);
}
@@ -1203,7 +1262,7 @@ pub mod tests {
let mut num = 0;
let mut found_key = false;
index.unchecked_scan_accounts(&ancestors, |pubkey, _index| {
index.unchecked_scan_accounts("", &ancestors, |pubkey, _index| {
if pubkey == &key.pubkey() {
found_key = true
};
@@ -1274,7 +1333,7 @@ pub mod tests {
let ancestors: Ancestors = HashMap::new();
let mut scanned_keys = HashSet::new();
index.range_scan_accounts(&ancestors, pubkey_range, |pubkey, _index| {
index.range_scan_accounts("", &ancestors, pubkey_range, |pubkey, _index| {
scanned_keys.insert(*pubkey);
});
@@ -1345,7 +1404,7 @@ pub mod tests {
let ancestors: Ancestors = HashMap::new();
let mut scanned_keys = HashSet::new();
index.unchecked_scan_accounts(&ancestors, |pubkey, _index| {
index.unchecked_scan_accounts("", &ancestors, |pubkey, _index| {
scanned_keys.insert(*pubkey);
});
assert_eq!(scanned_keys.len(), num_pubkeys);
@@ -1631,7 +1690,7 @@ pub mod tests {
let mut num = 0;
let mut found_key = false;
index.unchecked_scan_accounts(&Ancestors::new(), |pubkey, _index| {
index.unchecked_scan_accounts("", &Ancestors::new(), |pubkey, _index| {
if pubkey == &key.pubkey() {
found_key = true;
assert_eq!(_index, (&true, 3));