gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

clean_accounts calls AcctIdx: get_many (#20715)

Browse Source 
* AcctIdx: get_many

* keep read lock

* AcctIdx: get_many optionally adds to cache

* rename
This commit is contained in:
Jeff Washington (jwash)
2021-10-19 15:54:06 -05:00
committed by GitHub
parent 0acbfdfcb9
commit 47a58a38c2
2 changed files with 105 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
runtime/src/accounts_db.rs
View File

@ -2020,6 +2020,7 @@ impl AccountsDb {
let found_not_zero_accum = AtomicU64::new(0);
let not_found_on_fork_accum = AtomicU64::new(0);
let missing_accum = AtomicU64::new(0);
let useful_accum = AtomicU64::new(0);
// parallel scan the index.
let (mut purges_zero_lamports, purges_old_accounts) = {
@ -2032,51 +2033,71 @@ impl AccountsDb {
let mut found_not_zero = 0;
let mut not_found_on_fork = 0;
let mut missing = 0;
for pubkey in pubkeys {
match self.accounts_index.get(pubkey, None, max_clean_root) {
AccountIndexGetResult::Found(locked_entry, index) => {
let slot_list = locked_entry.slot_list();
let (slot, account_info) = &slot_list[index];
if account_info.lamports == 0 {
purges_zero_lamports.insert(
*pubkey,
self.accounts_index
.roots_and_ref_count(&locked_entry, max_clean_root),
);
} else {
found_not_zero += 1;
}
// Release the lock
let slot = *slot;
drop(locked_entry);
if uncleaned_roots.contains(&slot) {
// Assertion enforced by `accounts_index.get()`, the latest slot
// will not be greater than the given `max_clean_root`
if let Some(max_clean_root) = max_clean_root {
assert!(slot <= max_clean_root);
}
purges_old_accounts.push(*pubkey);
}
}
AccountIndexGetResult::NotFoundOnFork => {
// This pubkey is in the index but not in a root slot, so clean
// it up by adding it to the to-be-purged list.
//
// Also, this pubkey must have been touched by some slot since
// it was in the dirty list, so we assume that the slot it was
// touched in must be unrooted.
not_found_on_fork += 1;
purges_old_accounts.push(*pubkey);
}
AccountIndexGetResult::Missing(_lock) => {
let mut useful = 0;
self.accounts_index.scan(
pubkeys,
max_clean_root,
// return true if we want this item to remain in the cache
|exists, slot_list, index_in_slot_list, pubkey, ref_count| {
let mut useless = true;
if !exists {
missing += 1;
} else {
match index_in_slot_list {
Some(index_in_slot_list) => {
// found info relative to max_clean_root
let (slot, account_info) =
&slot_list[index_in_slot_list];
if account_info.lamports == 0 {
useless = false;
purges_zero_lamports.insert(
*pubkey,
(
self.accounts_index.get_rooted_entries(
slot_list,
max_clean_root,
),
ref_count,
),
);
} else {
found_not_zero += 1;
}
let slot = *slot;
if uncleaned_roots.contains(&slot) {
// Assertion enforced by `accounts_index.get()`, the latest slot
// will not be greater than the given `max_clean_root`
if let Some(max_clean_root) = max_clean_root {
assert!(slot <= max_clean_root);
}
purges_old_accounts.push(*pubkey);
useless = false;
}
}
None => {
// This pubkey is in the index but not in a root slot, so clean
// it up by adding it to the to-be-purged list.
//
// Also, this pubkey must have been touched by some slot since
// it was in the dirty list, so we assume that the slot it was
// touched in must be unrooted.
not_found_on_fork += 1;
useless = false;
purges_old_accounts.push(*pubkey);
}
}
}
};
}
if !useless {
useful += 1;
}
!useless
},
);
found_not_zero_accum.fetch_add(found_not_zero, Ordering::Relaxed);
not_found_on_fork_accum.fetch_add(not_found_on_fork, Ordering::Relaxed);
missing_accum.fetch_add(missing, Ordering::Relaxed);
useful_accum.fetch_add(useful, Ordering::Relaxed);
(purges_zero_lamports, purges_old_accounts)
})
.reduce(
@ -2234,6 +2255,7 @@ impl AccountsDb {
("delta_key_count", key_timings.delta_key_count, i64),
("dirty_pubkeys_count", key_timings.dirty_pubkeys_count, i64),
("sort_us", sort.as_us(), i64),
("useful_keys", useful_accum.load(Ordering::Relaxed), i64),
("total_keys_count", total_keys_count, i64),
(
"scan_found_not_zero",

43
runtime/src/accounts_index.rs
View File

@ -1400,6 +1400,49 @@ impl<T: IndexValue> AccountsIndex<T> {
self.storage.set_startup(value);
}
/// For each pubkey, find the latest account that appears in `roots` and <= `max_root`
/// call `callback`
pub(crate) fn scan<F>(&self, pubkeys: &[Pubkey], max_root: Option<Slot>, mut callback: F)
where
// return true if accounts index entry should be put in in_mem cache
// params:
// exists: false if not in index at all
// slot list found at slot at most max_root or empty slot list
// index in slot list where best slot was found or None if nothing found by root criteria
// pubkey looked up
// refcount of entry in index
F: FnMut(bool, &SlotList<T>, Option<usize>, &Pubkey, RefCount) -> bool,
{
let empty_slot_list = vec![];
let mut lock = None;
let mut last_bin = self.bins(); // too big, won't match
pubkeys.iter().for_each(|pubkey| {
let bin = self.bin_calculator.bin_from_pubkey(pubkey);
if bin != last_bin {
// cannot re-use lock since next pubkey is in a different bin than previous one
lock = Some(self.account_maps[bin].read().unwrap());
last_bin = bin;
}
lock.as_ref().unwrap().get_internal(pubkey, |entry| {
let cache = match entry {
Some(locked_entry) => {
let slot_list = &locked_entry.slot_list.read().unwrap();
let found_index = self.latest_slot(None, slot_list, max_root);
callback(
true,
slot_list,
found_index,
pubkey,
locked_entry.ref_count(),
)
}
None => callback(false, &empty_slot_list, None, pubkey, RefCount::MAX),
};
(cache, ())
});
});
}
/// Get an account
/// The latest account that appears in `ancestors` or `roots` is returned.
pub(crate) fn get(