gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Performance optimizations

Browse Source
This commit is contained in:
Sathish Ambley
2019-02-23 23:43:10 -08:00
committed by sakridge
parent 5216952691
commit 96b349dcbb
1 changed files with 163 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

249
runtime/src/accounts.rs
View File

@ -76,7 +76,7 @@ type AppendVecId = usize;
type Fork = u64; type Fork = u64;
struct AccountMap(HashMap<Fork, (AppendVecId, u64)>); struct AccountMap(RwLock<HashMap<Fork, (AppendVecId, u64)>>);
#[derive(Debug, PartialEq)] #[derive(Debug, PartialEq)]
enum AccountStorageStatus { enum AccountStorageStatus {
@ -176,7 +176,9 @@ fn cleanup_dirs(paths: &str) {
let paths = get_paths_vec(&paths); let paths = get_paths_vec(&paths);
paths.iter().for_each(|p| { paths.iter().for_each(|p| {
let _ignored = remove_dir_all(p); let _ignored = remove_dir_all(p);
}) let path = Path::new(p);
let _ignored = remove_dir_all(path.parent().unwrap());
});
} }
impl Drop for Accounts { impl Drop for Accounts {
@ -252,8 +254,9 @@ impl AccountsDB {
.unwrap() .unwrap()
.iter() .iter()
.for_each(|p| { .for_each(|p| {
if let Some(forks) = self.index_info.index.read().unwrap().get(p) { if let Some(entry) = self.index_info.index.read().unwrap().get(p) {
if let Some((id, index)) = forks.0.get(&fork) { let forks = entry.0.read().unwrap();
if let Some((id, index)) = forks.get(&fork) {
accounts.push( accounts.push(
self.storage.read().unwrap()[*id] self.storage.read().unwrap()[*id]
.appendvec .appendvec
@ -271,8 +274,9 @@ impl AccountsDB {
pub fn has_accounts(&self, fork: Fork) -> bool { pub fn has_accounts(&self, fork: Fork) -> bool {
let index = self.index_info.index.read().unwrap(); let index = self.index_info.index.read().unwrap();
for account_map in index.values() { for entry in index.values() {
if account_map.0.contains_key(&fork) { let account_map = entry.0.read().unwrap();
if account_map.contains_key(&fork) {
return true; return true;
} }
} }
@ -282,8 +286,9 @@ impl AccountsDB {
pub fn hash_internal_state(&self, fork: Fork) -> Option<Hash> { pub fn hash_internal_state(&self, fork: Fork) -> Option<Hash> {
let mut ordered_accounts = BTreeMap::new(); let mut ordered_accounts = BTreeMap::new();
let rindex = self.index_info.index.read().unwrap(); let rindex = self.index_info.index.read().unwrap();
rindex.iter().for_each(|(p, forks)| { rindex.iter().for_each(|(p, entry)| {
if let Some((id, index)) = forks.0.get(&fork) { let forks = entry.0.read().unwrap();
if let Some((id, index)) = forks.get(&fork) {
let account = self.storage.read().unwrap()[*id] let account = self.storage.read().unwrap()[*id]
.appendvec .appendvec
.read() .read()
@ -308,18 +313,21 @@ impl AccountsDB {
fn load(&self, fork: Fork, pubkey: &Pubkey, walk_back: bool) -> Option<Account> { fn load(&self, fork: Fork, pubkey: &Pubkey, walk_back: bool) -> Option<Account> {
let index = self.index_info.index.read().unwrap(); let index = self.index_info.index.read().unwrap();
if let Some(forks) = index.get(pubkey) { if let Some(map) = index.get(pubkey) {
let forks = map.0.read().unwrap();
// find most recent fork that is an ancestor of current_fork // find most recent fork that is an ancestor of current_fork
if let Some((id, offset)) = forks.0.get(&fork) { if let Some((id, offset)) = forks.get(&fork) {
return self.get_account(*id, *offset); return self.get_account(*id, *offset);
} else { } else {
if !walk_back { if !walk_back {
return None; return None;
} }
let fork_info = self.fork_info.read().unwrap(); let fork_info = self.fork_info.read().unwrap();
for parent_fork in fork_info.get(&fork).unwrap().parents.iter() { if let Some(info) = fork_info.get(&fork) {
if let Some((id, offset)) = forks.0.get(&parent_fork) { for parent_fork in info.parents.iter() {
return self.get_account(*id, *offset); if let Some((id, offset)) = forks.get(&parent_fork) {
return self.get_account(*id, *offset);
}
} }
} }
} }
@ -393,28 +401,42 @@ impl AccountsDB {
(id, offset) (id, offset)
} }
fn remove_account_entry(&self, fork: Fork, forks: &mut AccountMap) { fn remove_account_entries(&self, entries: &[Fork], map: &AccountMap) -> bool {
if let Some((id, _)) = forks.0.remove(&fork) { let mut forks = map.0.write().unwrap();
let stores = self.storage.read().unwrap(); for fork in entries.iter() {
if stores[id].count.fetch_sub(1, Ordering::Relaxed) == 1 { if let Some((id, _)) = forks.remove(&fork) {
stores[id].appendvec.write().unwrap().reset(); let stores = self.storage.read().unwrap();
stores[id].set_status(AccountStorageStatus::StorageAvailable); if stores[id].count.fetch_sub(1, Ordering::Relaxed) == 1 {
stores[id].appendvec.write().unwrap().reset();
stores[id].set_status(AccountStorageStatus::StorageAvailable);
}
}
}
forks.is_empty()
}
fn insert_account_entry(&self, fork: Fork, id: AppendVecId, offset: u64, map: &AccountMap) {
let mut forks = map.0.write().unwrap();
let stores = self.storage.read().unwrap();
stores[id].count.fetch_add(1, Ordering::Relaxed);
if let Some((old_id, _)) = forks.insert(fork, (id, offset)) {
if stores[old_id].count.fetch_sub(1, Ordering::Relaxed) == 1 {
stores[old_id].appendvec.write().unwrap().reset();
stores[old_id].set_status(AccountStorageStatus::StorageAvailable);
} }
} }
} }
/// Store the account update. If the update is to delete the account because the token balance /// Store the account update. If the update is to delete the account because
/// is 0, purge needs to be set to true for the delete to occur in place. /// the token balance is 0, purge needs to be set to true for the delete
pub fn store(&self, fork: Fork, purge: bool, pubkey: &Pubkey, account: &Account) { /// to occur in place.
fn store_account(&self, fork: Fork, purge: bool, pubkey: &Pubkey, account: &Account) {
if account.tokens == 0 && purge { if account.tokens == 0 && purge {
println!("store purge {} {:?}", fork, pubkey);
// purge if balance is 0 and no checkpoints // purge if balance is 0 and no checkpoints
let mut index = self.index_info.index.write().unwrap(); let index = self.index_info.index.read().unwrap();
if let Some(mut forks) = index.get_mut(&pubkey) { let map = index.get(&pubkey).unwrap();
self.remove_account_entry(fork, &mut forks); self.remove_account_entries(&[fork], &map);
if forks.0.is_empty() {
index.remove(pubkey);
}
}
if vote_program::check_id(&account.owner) { if vote_program::check_id(&account.owner) {
self.index_info.vote_index.write().unwrap().remove(pubkey); self.index_info.vote_index.write().unwrap().remove(pubkey);
} }
@ -430,21 +452,20 @@ impl AccountsDB {
} }
} }
let result: Option<(AppendVecId, u64)>; let index = self.index_info.index.read().unwrap();
{ let map = index.get(&pubkey).unwrap();
self.insert_account_entry(fork, id, offset, &map);
}
}
pub fn store(&self, fork: Fork, purge: bool, pubkey: &Pubkey, account: &Account) {
{
if !self.index_info.index.read().unwrap().contains_key(&pubkey) {
let mut windex = self.index_info.index.write().unwrap(); let mut windex = self.index_info.index.write().unwrap();
let forks = windex.entry(*pubkey).or_insert(AccountMap(HashMap::new())); windex.insert(*pubkey, AccountMap(RwLock::new(HashMap::new())));
result = forks.0.insert(fork, (id, offset));
}
let stores = self.storage.read().unwrap();
stores[id].count.fetch_add(1, Ordering::Relaxed);
if let Some((old_id, _)) = result {
if stores[old_id].count.fetch_sub(1, Ordering::Relaxed) == 1 {
stores[old_id].appendvec.write().unwrap().reset();
stores[old_id].set_status(AccountStorageStatus::StorageAvailable);
}
} }
} }
self.store_account(fork, purge, pubkey, account);
} }
pub fn store_accounts( pub fn store_accounts(
@ -455,6 +476,27 @@ impl AccountsDB {
res: &[Result<()>], res: &[Result<()>],
loaded: &[Result<(InstructionAccounts, InstructionLoaders)>], loaded: &[Result<(InstructionAccounts, InstructionLoaders)>],
) { ) {
let mut keys = vec![];
{
let index = self.index_info.index.read().unwrap();
for (i, raccs) in loaded.iter().enumerate() {
if res[i].is_err() || raccs.is_err() {
continue;
}
let tx = &txs[i];
for key in tx.account_keys.iter() {
if !index.contains_key(&key) {
keys.push(*key);
}
}
}
}
if !keys.is_empty() {
let mut index = self.index_info.index.write().unwrap();
for key in keys.iter() {
index.insert(*key, AccountMap(RwLock::new(HashMap::new())));
}
}
for (i, raccs) in loaded.iter().enumerate() { for (i, raccs) in loaded.iter().enumerate() {
if res[i].is_err() || raccs.is_err() { if res[i].is_err() || raccs.is_err() {
continue; continue;
@ -598,14 +640,36 @@ impl AccountsDB {
} }
} }
fn remove_parents(&self, fork: Fork) -> Vec<Fork> {
let mut info = self.fork_info.write().unwrap();
let fork_info = info.get_mut(&fork).unwrap();
let parents = fork_info.parents.split_off(0);
info.retain(|&f, _| !parents.contains(&f));
parents
}
fn get_merged_index(
&self,
fork: Fork,
parents: &[Fork],
map: &AccountMap,
) -> Option<(Fork, AppendVecId, u64)> {
let forks = map.0.read().unwrap();
if let Some((id, offset)) = forks.get(&fork) {
return Some((fork, *id, *offset));
} else {
for parent_fork in parents.iter() {
if let Some((id, offset)) = forks.get(parent_fork) {
return Some((*parent_fork, *id, *offset));
}
}
}
None
}
/// become the root accountsDB /// become the root accountsDB
fn squash(&self, fork: Fork) { fn squash(&self, fork: Fork) {
let parents: Vec<Fork>; let parents = self.remove_parents(fork);
{
let info = self.fork_info.read().unwrap();
let fork_info = info.get(&fork).unwrap();
parents = fork_info.parents.clone();
}
let tx_count = parents let tx_count = parents
.iter() .iter()
.fold(0, |sum, parent| sum + self.transaction_count(*parent)); .fold(0, |sum, parent| sum + self.transaction_count(*parent));
@ -613,41 +677,37 @@ impl AccountsDB {
// for every account in all the parents, load latest and update self if // for every account in all the parents, load latest and update self if
// absent // absent
let mut index = self.index_info.index.write().unwrap(); let mut keys = vec![];
index.iter_mut().for_each(|(_, mut forks)| { {
if forks.0.get(&fork).is_none() { let index = self.index_info.index.read().unwrap();
for parent_fork in parents.iter() { index.iter().for_each(|(pubkey, map)| {
if let Some((id, offset)) = forks.0.get(parent_fork) { if let Some((parent_fork, id, offset)) = self.get_merged_index(fork, &parents, &map)
forks.0.insert(fork, (*id, *offset)); {
forks.0.remove(parent_fork); if parent_fork != fork {
break; self.insert_account_entry(fork, id, offset, &map);
if self.remove_account_entries(&parents, &map) {
keys.push(pubkey.clone());
}
}
let account = self.get_account(id, offset).unwrap();
if account.tokens == 0 {
if self.remove_account_entries(&[fork], &map) {
keys.push(pubkey.clone());
}
if vote_program::check_id(&account.owner) {
self.index_info.vote_index.write().unwrap().remove(pubkey);
}
} else {
} }
} }
} });
for parent_fork in parents.iter() { }
self.remove_account_entry(*parent_fork, &mut forks); if !keys.is_empty() {
} let mut index = self.index_info.index.write().unwrap();
}); for key in keys.iter() {
index.remove(&key);
{
let mut info = self.fork_info.write().unwrap();
let fork_info = info.get_mut(&fork).unwrap();
fork_info.parents = vec![];
for parent_fork in parents.iter() {
info.remove(parent_fork);
} }
} }
index.iter_mut().for_each(|(pubkey, mut forks)| {
if let Some((id, offset)) = forks.0.get(&fork) {
let account = self.get_account(*id, *offset).unwrap();
if account.tokens == 0 {
self.remove_account_entry(fork, &mut forks);
if vote_program::check_id(&account.owner) {
self.index_info.vote_index.write().unwrap().remove(pubkey);
}
}
}
});
} }
} }
@ -710,7 +770,7 @@ impl Accounts {
/// * purge - if the account token value is 0 and purge is true then delete the account. /// * purge - if the account token value is 0 and purge is true then delete the account.
/// purge should be set to false for overlays, and true for the root checkpoint. /// purge should be set to false for overlays, and true for the root checkpoint.
pub fn store_slow(&self, fork: Fork, purge: bool, pubkey: &Pubkey, account: &Account) { pub fn store_slow(&self, fork: Fork, purge: bool, pubkey: &Pubkey, account: &Account) {
self.accounts_db.store(fork, purge, pubkey, account) self.accounts_db.store(fork, purge, pubkey, account);
} }
fn lock_account( fn lock_account(
@ -1254,26 +1314,43 @@ mod tests {
#[test] #[test]
fn test_accountsdb_squash() { fn test_accountsdb_squash() {
let paths = "squash".to_string(); let paths = "squash".to_string();
let db0 = AccountsDB::new(0, &paths); let db = AccountsDB::new(0, &paths);
let key = Pubkey::default(); let key = Pubkey::default();
let account0 = Account::new(1, 0, key); let account0 = Account::new(1, 0, key);
// store value 1 in the "root", i.e. db zero // store value 1 in the "root", i.e. db zero
db0.store(0, true, &key, &account0); db.store(0, true, &key, &account0);
db0.add_fork(1, Some(0)); let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&db, &mut pubkeys, 100, 0);
for _ in 1..100 {
let idx = thread_rng().gen_range(0, 99);
let account = db.load(0, &pubkeys[idx], true).unwrap();
let mut default_account = Account::default();
default_account.tokens = (idx + 1) as u64;
assert_eq!(compare_account(&default_account, &account), true);
}
db.add_fork(1, Some(0));
// store value 0 in the child, but don't purge (see purge test above) // store value 0 in the child, but don't purge (see purge test above)
let account1 = Account::new(0, 0, key); let account1 = Account::new(0, 0, key);
db0.store(1, false, &key, &account1); db.store(1, false, &key, &account1);
// masking accounts is done at the Accounts level, at accountsDB we see // masking accounts is done at the Accounts level, at accountsDB we see
// original account // original account
assert_eq!(db0.load(1, &key, true), Some(account1)); assert_eq!(db.load(1, &key, true), Some(account1));
// merge, which should whack key's account // merge, which should whack key's account
db0.squash(1); db.squash(1);
assert_eq!(db0.load(1, &key, true), None); assert_eq!(db.load(1, &key, true), None);
for _ in 1..100 {
let idx = thread_rng().gen_range(0, 99);
assert_eq!(db.load(0, &pubkeys[idx], true), None);
let account = db.load(1, &pubkeys[idx], true).unwrap();
let mut default_account = Account::default();
default_account.tokens = (idx + 1) as u64;
assert_eq!(compare_account(&default_account, &account), true);
}
cleanup_dirs(&paths); cleanup_dirs(&paths);
} }