gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add Confirmations parameter to RPC Subscriptions (#4154)

Browse Source 
* Add optional depth parameter to pubsub, and store in subscriptions

* Pass bank_forks into rpc_subscription; add method to check depth before notify and impl for account subscriptions

* Impl check-depth for signature subscriptions

* Impl check-depth for program subscriptions

* Plumb fork id through accounts

* Use fork id and root to prevent repeated account notifications; also s/Depth/Confirmations

* Write tests in terms of bank_forks

* Fixup accounts tests

* Add pubsub-confirmations tests

* Update pubsub documentation
This commit is contained in:
Tyera Eulberg
2019-05-06 08:31:50 -06:00
committed by GitHub
parent 0139e5db21
commit 71f9b44687
8 changed files with 430 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
runtime/src/accounts.rs
View File

@@ -163,7 +163,9 @@ impl Accounts {
let mut called_accounts: Vec<Account> = vec![];
for key in &message.account_keys {
called_accounts.push(
AccountsDB::load(storage, ancestors, accounts_index, key).unwrap_or_default(),
AccountsDB::load(storage, ancestors, accounts_index, key)
.map(|(account, _)| account)
.unwrap_or_default(),
);
}
if called_accounts.is_empty() || called_accounts[0].lamports == 0 {
@@ -201,7 +203,9 @@ impl Accounts {
}
depth += 1;
let program = match AccountsDB::load(storage, ancestors, accounts_index, &program_id) {
let program = match AccountsDB::load(storage, ancestors, accounts_index, &program_id)
.map(|(account, _)| account)
{
Some(program) => program,
None => {
error_counters.account_not_found += 1;
@@ -289,10 +293,14 @@ impl Accounts {
}
/// Slow because lock is held for 1 operation instead of many
pub fn load_slow(&self, ancestors: &HashMap<Fork, usize>, pubkey: &Pubkey) -> Option<Account> {
pub fn load_slow(
&self,
ancestors: &HashMap<Fork, usize>,
pubkey: &Pubkey,
) -> Option<(Account, Fork)> {
self.accounts_db
.load_slow(ancestors, pubkey)
.filter(|acc| acc.lamports != 0)
.filter(|(acc, _)| acc.lamports != 0)
}
pub fn load_by_program(&self, fork: Fork, program_id: &Pubkey) -> Vec<(Pubkey, Account)> {

70
runtime/src/accounts_db.rs
View File

@@ -259,15 +259,20 @@ impl AccountsDB {
ancestors: &HashMap<Fork, usize>,
accounts_index: &AccountsIndex<AccountInfo>,
pubkey: &Pubkey,
) -> Option<Account> {
let info = accounts_index.get(pubkey, ancestors)?;
) -> Option<(Account, Fork)> {
let (info, fork) = accounts_index.get(pubkey, ancestors)?;
//TODO: thread this as a ref
storage
.get(&info.id)
.and_then(|store| Some(store.accounts.get_account(info.offset)?.0.clone_account()))
.map(|account| (account, fork))
}
pub fn load_slow(&self, ancestors: &HashMap<Fork, usize>, pubkey: &Pubkey) -> Option<Account> {
pub fn load_slow(
&self,
ancestors: &HashMap<Fork, usize>,
pubkey: &Pubkey,
) -> Option<(Account, Fork)> {
let accounts_index = self.accounts_index.read().unwrap();
let storage = self.storage.read().unwrap();
Self::load(&storage, ancestors, &accounts_index, pubkey)
@@ -480,7 +485,7 @@ mod tests {
db.store(0, &[(&key, &account0)]);
db.add_root(0);
let ancestors = vec![(1, 1)].into_iter().collect();
assert_eq!(db.load_slow(&ancestors, &key), Some(account0));
assert_eq!(db.load_slow(&ancestors, &key), Some((account0, 0)));
}
#[test]
@@ -497,10 +502,10 @@ mod tests {
db.store(1, &[(&key, &account1)]);
let ancestors = vec![(1, 1)].into_iter().collect();
assert_eq!(&db.load_slow(&ancestors, &key).unwrap(), &account1);
assert_eq!(&db.load_slow(&ancestors, &key).unwrap().0, &account1);
let ancestors = vec![(1, 1), (0, 0)].into_iter().collect();
assert_eq!(&db.load_slow(&ancestors, &key).unwrap(), &account1);
assert_eq!(&db.load_slow(&ancestors, &key).unwrap().0, &account1);
}
#[test]
@@ -518,10 +523,10 @@ mod tests {
db.add_root(0);
let ancestors = vec![(1, 1)].into_iter().collect();
assert_eq!(&db.load_slow(&ancestors, &key).unwrap(), &account1);
assert_eq!(&db.load_slow(&ancestors, &key).unwrap().0, &account1);
let ancestors = vec![(1, 1), (0, 0)].into_iter().collect();
assert_eq!(&db.load_slow(&ancestors, &key).unwrap(), &account1);
assert_eq!(&db.load_slow(&ancestors, &key).unwrap().0, &account1);
}
#[test]
@@ -552,18 +557,18 @@ mod tests {
// original account (but could also accept "None", which is implemented
// at the Accounts level)
let ancestors = vec![(0, 0), (1, 1)].into_iter().collect();
assert_eq!(&db.load_slow(&ancestors, &key).unwrap(), &account1);
assert_eq!(&db.load_slow(&ancestors, &key).unwrap().0, &account1);
// we should see 1 token in fork 2
let ancestors = vec![(0, 0), (2, 2)].into_iter().collect();
assert_eq!(&db.load_slow(&ancestors, &key).unwrap(), &account0);
assert_eq!(&db.load_slow(&ancestors, &key).unwrap().0, &account0);
db.add_root(0);
let ancestors = vec![(1, 1)].into_iter().collect();
assert_eq!(db.load_slow(&ancestors, &key), Some(account1));
assert_eq!(db.load_slow(&ancestors, &key), Some((account1, 1)));
let ancestors = vec![(2, 2)].into_iter().collect();
assert_eq!(db.load_slow(&ancestors, &key), Some(account0)); // original value
assert_eq!(db.load_slow(&ancestors, &key), Some((account0, 0))); // original value
}
#[test]
@@ -579,7 +584,7 @@ mod tests {
let account = db.load_slow(&ancestors, &pubkeys[idx]).unwrap();
let mut default_account = Account::default();
default_account.lamports = (idx + 1) as u64;
assert_eq!(default_account, account);
assert_eq!((default_account, 0), account);
}
db.add_root(0);
@@ -593,8 +598,8 @@ mod tests {
let account1 = db.load_slow(&ancestors, &pubkeys[idx]).unwrap();
let mut default_account = Account::default();
default_account.lamports = (idx + 1) as u64;
assert_eq!(&default_account, &account0);
assert_eq!(&default_account, &account1);
assert_eq!(&default_account, &account0.0);
assert_eq!(&default_account, &account1.0);
}
}
@@ -650,9 +655,9 @@ mod tests {
// masking accounts is done at the Accounts level, at accountsDB we see
// original account
let ancestors = vec![(0, 0), (1, 1)].into_iter().collect();
assert_eq!(db0.load_slow(&ancestors, &key), Some(account1));
assert_eq!(db0.load_slow(&ancestors, &key), Some((account1, 1)));
let ancestors = vec![(0, 0)].into_iter().collect();
assert_eq!(db0.load_slow(&ancestors, &key), Some(account0));
assert_eq!(db0.load_slow(&ancestors, &key), Some((account0, 0)));
}
fn create_account(
@@ -685,7 +690,7 @@ mod tests {
for _ in 1..1000 {
let idx = thread_rng().gen_range(0, range);
let ancestors = vec![(fork, 0)].into_iter().collect();
if let Some(mut account) = accounts.load_slow(&ancestors, &pubkeys[idx]) {
if let Some((mut account, _)) = accounts.load_slow(&ancestors, &pubkeys[idx]) {
account.lamports = account.lamports + 1;
accounts.store(fork, &[(&pubkeys[idx], &account)]);
if account.lamports == 0 {
@@ -714,7 +719,7 @@ mod tests {
let account = accounts.load_slow(&ancestors, &pubkeys[idx]).unwrap();
let mut default_account = Account::default();
default_account.lamports = (idx + 1) as u64;
assert_eq!(default_account, account);
assert_eq!((default_account, 0), account);
}
}
@@ -728,7 +733,7 @@ mod tests {
let account = accounts.load_slow(&ancestors, &pubkeys[0]).unwrap();
let mut default_account = Account::default();
default_account.lamports = 1;
assert_eq!(default_account, account);
assert_eq!((default_account, 0), account);
}
#[test]
@@ -765,7 +770,7 @@ mod tests {
for (i, key) in keys.iter().enumerate() {
let ancestors = vec![(0, 0)].into_iter().collect();
assert_eq!(
accounts.load_slow(&ancestors, &key).unwrap().lamports,
accounts.load_slow(&ancestors, &key).unwrap().0.lamports,
(i as u64) + 1
);
}
@@ -810,8 +815,14 @@ mod tests {
assert_eq!(stores[&1].status(), AccountStorageStatus::StorageAvailable);
}
let ancestors = vec![(0, 0)].into_iter().collect();
assert_eq!(accounts.load_slow(&ancestors, &pubkey1).unwrap(), account1);
assert_eq!(accounts.load_slow(&ancestors, &pubkey2).unwrap(), account2);
assert_eq!(
accounts.load_slow(&ancestors, &pubkey1).unwrap().0,
account1
);
assert_eq!(
accounts.load_slow(&ancestors, &pubkey2).unwrap().0,
account2
);
// lots of stores, but 3 storages should be enough for everything
for i in 0..25 {
@@ -828,8 +839,14 @@ mod tests {
assert_eq!(stores[&2].status(), status[0]);
}
let ancestors = vec![(0, 0)].into_iter().collect();
assert_eq!(accounts.load_slow(&ancestors, &pubkey1).unwrap(), account1);
assert_eq!(accounts.load_slow(&ancestors, &pubkey2).unwrap(), account2);
assert_eq!(
accounts.load_slow(&ancestors, &pubkey1).unwrap().0,
account1
);
assert_eq!(
accounts.load_slow(&ancestors, &pubkey2).unwrap().0,
account2
);
}
}
@@ -875,6 +892,7 @@ mod tests {
.unwrap()
.get(&pubkey, &ancestors)
.unwrap()
.0
.clone();
//fork 0 is behind root, but it is not root, therefore it is purged
accounts.add_root(1);
@@ -891,7 +909,7 @@ mod tests {
//new value is there
let ancestors = vec![(1, 1)].into_iter().collect();
assert_eq!(accounts.load_slow(&ancestors, &pubkey), Some(account));;
assert_eq!(accounts.load_slow(&ancestors, &pubkey), Some((account, 1)));
}
}

20
runtime/src/accounts_index.rs
View File

@@ -15,21 +15,21 @@ pub struct AccountsIndex<T> {
impl<T: Clone> AccountsIndex<T> {
/// Get an account
/// The latest account that appears in `ancestors` or `roots` is returned.
pub fn get(&self, pubkey: &Pubkey, ancestors: &HashMap<Fork, usize>) -> Option<&T> {
pub fn get(&self, pubkey: &Pubkey, ancestors: &HashMap<Fork, usize>) -> Option<(&T, Fork)> {
let list = self.account_maps.get(pubkey)?;
let mut max = 0;
let mut rv = None;
for e in list.iter().rev() {
if e.0 >= max && (ancestors.get(&e.0).is_some() || self.is_root(e.0)) {
trace!("GET {} {:?}", e.0, ancestors);
rv = Some(&e.1);
rv = Some((&e.1, e.0));
max = e.0;
}
}
rv
}
/// Insert a new fork.
/// Insert a new fork.
/// @retval - The return value contains any squashed accounts that can freed from storage.
pub fn insert(&mut self, fork: Fork, pubkey: &Pubkey, account_info: T) -> Vec<(Fork, T)> {
let mut rv = vec![];
@@ -123,7 +123,7 @@ mod tests {
assert!(gc.is_empty());
let ancestors = vec![(0, 0)].into_iter().collect();
assert_eq!(index.get(&key.pubkey(), &ancestors), Some(&true));
assert_eq!(index.get(&key.pubkey(), &ancestors), Some((&true, 0)));
}
#[test]
@@ -143,7 +143,7 @@ mod tests {
let ancestors = vec![].into_iter().collect();
index.add_root(0);
assert_eq!(index.get(&key.pubkey(), &ancestors), Some(&true));
assert_eq!(index.get(&key.pubkey(), &ancestors), Some((&true, 0)));
}
#[test]
@@ -199,11 +199,11 @@ mod tests {
let ancestors = vec![(0, 0)].into_iter().collect();
let gc = index.insert(0, &key.pubkey(), true);
assert!(gc.is_empty());
assert_eq!(index.get(&key.pubkey(), &ancestors), Some(&true));
assert_eq!(index.get(&key.pubkey(), &ancestors), Some((&true, 0)));
let gc = index.insert(0, &key.pubkey(), false);
assert_eq!(gc, vec![(0, true)]);
assert_eq!(index.get(&key.pubkey(), &ancestors), Some(&false));
assert_eq!(index.get(&key.pubkey(), &ancestors), Some((&false, 0)));
}
#[test]
@@ -215,9 +215,9 @@ mod tests {
assert!(gc.is_empty());
let gc = index.insert(1, &key.pubkey(), false);
assert!(gc.is_empty());
assert_eq!(index.get(&key.pubkey(), &ancestors), Some(&true));
assert_eq!(index.get(&key.pubkey(), &ancestors), Some((&true, 0)));
let ancestors = vec![(1, 0)].into_iter().collect();
assert_eq!(index.get(&key.pubkey(), &ancestors), Some(&false));
assert_eq!(index.get(&key.pubkey(), &ancestors), Some((&false, 1)));
}
#[test]
@@ -230,6 +230,6 @@ mod tests {
let gc = index.insert(1, &key.pubkey(), false);
assert_eq!(gc, vec![(0, true)]);
let ancestors = vec![].into_iter().collect();
assert_eq!(index.get(&key.pubkey(), &ancestors), Some(&false));
assert_eq!(index.get(&key.pubkey(), &ancestors), Some((&false, 1)));
}
}

7
runtime/src/bank.rs
View File

@@ -5,6 +5,7 @@
use crate::accounts::Accounts;
use crate::accounts_db::{ErrorCounters, InstructionAccounts, InstructionLoaders};
use crate::accounts_index::Fork;
use crate::blockhash_queue::BlockhashQueue;
use crate::locked_accounts_results::LockedAccountsResults;
use crate::message_processor::{MessageProcessor, ProcessInstruction};
@@ -888,7 +889,9 @@ impl Bank {
}
pub fn get_account(&self, pubkey: &Pubkey) -> Option<Account> {
self.accounts.load_slow(&self.ancestors, pubkey)
self.accounts
.load_slow(&self.ancestors, pubkey)
.map(|(account, _)| account)
}
pub fn get_program_accounts_modified_since_parent(
@@ -898,7 +901,7 @@ impl Bank {
self.accounts.load_by_program(self.slot(), program_id)
}
pub fn get_account_modified_since_parent(&self, pubkey: &Pubkey) -> Option<Account> {
pub fn get_account_modified_since_parent(&self, pubkey: &Pubkey) -> Option<(Account, Fork)> {
let just_self: HashMap<u64, usize> = vec![(self.slot(), 0)].into_iter().collect();
self.accounts.load_slow(&just_self, pubkey)
}