diff --git a/runtime/src/accounts.rs b/runtime/src/accounts.rs index c77f7b31b3..f751ed0c1a 100644 --- a/runtime/src/accounts.rs +++ b/runtime/src/accounts.rs @@ -255,13 +255,29 @@ impl AccountsDB { pub fn transaction_count(&self) -> u64 { self.transaction_count } - //pub fn account_values_slow(&self) -> Vec<(Pubkey, solana_sdk::account::Account)> { - // self.accounts.iter().map(|(x, y)| (*x, y.clone())).collect() - //} - //fn merge(&mut self, other: Self) { - // self.transaction_count += other.transaction_count; - // self.accounts.extend(other.accounts) - //} + + /// become the root accountsDB + fn merge_parents(&mut self, parents: &[U]) + where + U: Deref, + { + self.transaction_count += parents + .iter() + .fold(0, |sum, parent| sum + parent.transaction_count); + + // for every account in all the parents, load latest and update self if + // absent + for pubkey in parents.iter().flat_map(|parent| parent.accounts.keys()) { + // update self with data from parents unless in self + if self.accounts.get(pubkey).is_none() { + self.accounts + .insert(pubkey.clone(), Self::load(parents, pubkey).unwrap().clone()); + } + } + + // toss any zero-balance accounts, since self is root now + self.accounts.retain(|_, account| account.tokens != 0); + } } impl Accounts { @@ -389,27 +405,22 @@ impl Accounts { pub fn transaction_count(&self) -> u64 { self.accounts_db.read().unwrap().transaction_count() } - ///// accounts starts with an empty data structure for every fork - ///// self is root, merge the fork into self - //pub fn merge_into_root(&self, other: Self) { - // assert!(other.account_locks.lock().unwrap().is_empty()); - // let db = other.accounts_db.into_inner().unwrap(); - // let mut mydb = self.accounts_db.write().unwrap(); - // mydb.merge(db) - //} - //pub fn copy_for_tpu(&self) -> Self { - // //TODO: deprecate this in favor of forks and merge_into_root - // let copy = Accounts::default(); - // { - // let mut accounts_db = copy.accounts_db.write().unwrap(); - // for (key, val) in self.accounts_db.read().unwrap().accounts.iter() { - // accounts_db.accounts.insert(key.clone(), val.clone()); - // } - // accounts_db.transaction_count = self.transaction_count(); - // } - // copy - //} + /// accounts starts with an empty data structure for every child/fork + /// this merges all the parents/checkpoints + pub fn merge_parents(&self, parents: &[U]) + where + U: Deref, + { + assert!(self.account_locks.lock().unwrap().is_empty()); + + let dbs: Vec<_> = parents + .iter() + .map(|obj| obj.accounts_db.read().unwrap()) + .collect(); + + self.accounts_db.write().unwrap().merge_parents(&dbs); + } } #[cfg(test)] @@ -819,4 +830,24 @@ mod tests { loaded_accounts[0].clone().unwrap_err(); assert_eq!(loaded_accounts[0], Err(BankError::AccountLoadedTwice)); } + + #[test] + fn test_accounts_merge_parents() { + let mut db0 = AccountsDB::default(); + let key = Pubkey::default(); + let account = Account::new(1, 0, key); + + // store value 1 in the "root", i.e. db zero + db0.store(true, &key, &account); + + // store value 0 in the child, but don't purge (see purge test above) + let mut db1 = AccountsDB::default(); + db1.store(false, &key, &Account::new(0, 0, key)); + + // merge, which should whack key's account + db1.merge_parents(&[&db0]); + + assert_eq!(AccountsDB::load(&[&db1], &key), None); + } + } diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index f2f7c3fc53..d9bd6b83a0 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -135,10 +135,33 @@ impl Bank { bank } + /// merge (i.e. pull) the parent's state up into this Bank, + /// this Bank becomes a root + pub fn merge_parents(&mut self) { + let parents = self.parents(); + self.parent = None; + + let parent_accounts: Vec<_> = parents.iter().map(|b| &b.accounts).collect(); + self.accounts.merge_parents(&parent_accounts); + + let parent_caches: Vec<_> = parents + .iter() + .map(|b| b.status_cache.read().unwrap()) + .collect(); + self.status_cache + .write() + .unwrap() + .merge_parents(&parent_caches); + } + /// Return the more recent checkpoint of this bank instance. pub fn parent(&self) -> Option> { self.parent.clone() } + /// Returns whether this bank is the root + pub fn is_root(&self) -> bool { + self.parent.is_none() + } fn process_genesis_block(&self, genesis_block: &GenesisBlock) { assert!(genesis_block.mint_id != Pubkey::default()); @@ -172,7 +195,7 @@ impl Bank { .unwrap(); self.accounts.store_slow( - true, + self.is_root(), &genesis_block.bootstrap_leader_vote_account_id, &bootstrap_leader_vote_account, ); @@ -185,7 +208,8 @@ impl Bank { pub fn add_native_program(&self, name: &str, program_id: &Pubkey) { let account = native_loader::create_program_account(name); - self.accounts.store_slow(true, program_id, &account); + self.accounts + .store_slow(self.is_root(), program_id, &account); } fn add_builtin_programs(&self) { @@ -197,8 +221,11 @@ impl Bank { self.add_native_program("solana_erc20", &token_program::id()); let storage_system_account = Account::new(1, 16 * 1024, storage_program::system_id()); - self.accounts - .store_slow(true, &storage_program::system_id(), &storage_system_account); + self.accounts.store_slow( + self.is_root(), + &storage_program::system_id(), + &storage_system_account, + ); } /// Return the last entry ID registered. @@ -466,7 +493,7 @@ impl Bank { ) { let now = Instant::now(); self.accounts - .store_accounts(true, txs, executed, loaded_accounts); + .store_accounts(self.is_root(), txs, executed, loaded_accounts); // once committed there is no way to unroll let write_elapsed = now.elapsed(); @@ -546,7 +573,7 @@ impl Bank { pub fn deposit(&self, pubkey: &Pubkey, tokens: u64) { let mut account = self.get_account(pubkey).unwrap_or_default(); account.tokens += tokens; - self.accounts.store_slow(true, pubkey, &account); + self.accounts.store_slow(self.is_root(), pubkey, &account); } pub fn get_account(&self, pubkey: &Pubkey) -> Option { @@ -1170,4 +1197,48 @@ mod tests { let bank1 = Bank::new(&GenesisBlock::new(20).0); assert_ne!(bank0.hash_internal_state(), bank1.hash_internal_state()); } + + /// Verifies that last ids and accounts are correctly referenced from parent + #[test] + fn test_bank_merge_parents() { + let (genesis_block, mint_keypair) = GenesisBlock::new(2); + let key1 = Keypair::new(); + let key2 = Keypair::new(); + let parent = Arc::new(Bank::new(&genesis_block)); + + let tx_move_mint_to_1 = SystemTransaction::new_move( + &mint_keypair, + key1.pubkey(), + 1, + genesis_block.last_id(), + 0, + ); + assert_eq!(parent.process_transaction(&tx_move_mint_to_1), Ok(())); + let mut bank = Bank::new_from_parent(&parent); + let tx_move_1_to_2 = + SystemTransaction::new_move(&key1, key2.pubkey(), 1, genesis_block.last_id(), 0); + assert_eq!(bank.process_transaction(&tx_move_1_to_2), Ok(())); + assert_eq!( + parent.get_signature_status(&tx_move_1_to_2.signatures[0]), + None + ); + + for _ in 0..3 { + // first time these should match what happened above, assert that parents are ok + assert_eq!(bank.get_balance(&key1.pubkey()), 0); + assert_eq!(bank.get_balance(&key2.pubkey()), 1); + assert_eq!( + bank.get_signature_status(&tx_move_mint_to_1.signatures[0]), + Some(Ok(())) + ); + assert_eq!( + bank.get_signature_status(&tx_move_1_to_2.signatures[0]), + Some(Ok(())) + ); + + // works iteration 0, no-ops on iteration 1 and 2 + bank.merge_parents(); + } + } + } diff --git a/runtime/src/last_id_queue.rs b/runtime/src/last_id_queue.rs index 5223445291..81d28402bb 100644 --- a/runtime/src/last_id_queue.rs +++ b/runtime/src/last_id_queue.rs @@ -123,23 +123,6 @@ impl LastIdQueue { self.tick_height = 0; self.last_id = None; } - /// fork for LastIdQueue is a simple clone - #[cfg(test)] - pub fn fork(&self) -> Self { - Self { - entries: self.entries.clone(), - tick_height: self.tick_height, - last_id: self.last_id, - } - } - /// merge for entryq is a swap - #[cfg(test)] - pub fn merge_into_root(&mut self, other: Self) { - let (entries, tick_height, last_id) = { (other.entries, other.tick_height, other.last_id) }; - self.entries = entries; - self.tick_height = tick_height; - self.last_id = last_id; - } } #[cfg(test)] mod tests { @@ -166,23 +149,4 @@ mod tests { // Assert we're no longer able to use the oldest entry ID. assert!(!entry_queue.check_entry(last_id)); } - #[test] - fn test_fork() { - let last_id = Hash::default(); - let mut first = LastIdQueue::default(); - assert!(!first.check_entry(last_id)); - first.register_tick(&last_id); - let second = first.fork(); - assert!(second.check_entry(last_id)); - } - #[test] - fn test_merge() { - let last_id = Hash::default(); - let mut first = LastIdQueue::default(); - assert!(!first.check_entry(last_id)); - let mut second = first.fork(); - second.register_tick(&last_id); - first.merge_into_root(second); - assert!(first.check_entry(last_id)); - } } diff --git a/runtime/src/status_cache.rs b/runtime/src/status_cache.rs index 08f2a2a99b..fe49f4132e 100644 --- a/runtime/src/status_cache.rs +++ b/runtime/src/status_cache.rs @@ -83,17 +83,37 @@ impl StatusCache { } self.get_signature_status_merged(sig) } - /// like accounts, status cache starts with an new data structure for every checkpoint - /// so only merge is implemented - /// but the merges maintains a history - #[cfg(test)] - pub fn merge_into_root(&mut self, other: Self) { - // merges should be empty for every other checkpoint accept the root - // which cannot be rolled back - assert!(other.merges.is_empty()); - self.merges.push_front(other); - if self.merges.len() > MAX_CACHE_ENTRIES { - self.merges.pop_back(); + + fn merge_parent_is_full(&mut self, parent: &Self) -> bool { + // flatten the parent and their merges into self.merges, limit + + self.merges.push_back(StatusCache { + signatures: parent.signatures.clone(), + failures: parent.failures.clone(), + merges: VecDeque::new(), + }); + for merge in &parent.merges { + self.merges.push_back(StatusCache { + signatures: merge.signatures.clone(), + failures: merge.failures.clone(), + merges: VecDeque::new(), + }); + } + self.merges.truncate(MAX_CACHE_ENTRIES); + + self.merges.len() == MAX_CACHE_ENTRIES + } + + /// copy the parents and parents' merges up to this instance, up to + /// MAX_CACHE_ENTRIES deep + pub fn merge_parents(&mut self, parents: &[U]) + where + U: Deref, + { + for parent in parents { + if self.merge_parent_is_full(parent) { + break; + } } } @@ -158,10 +178,10 @@ mod tests { let last_id = hash(Hash::default().as_ref()); let mut status_cache = BankStatusCache::new(&last_id); assert_eq!(status_cache.has_signature(&sig), false); - assert_eq!(status_cache.get_signature_status(&sig), None,); + assert_eq!(status_cache.get_signature_status(&sig), None); status_cache.add(&sig); assert_eq!(status_cache.has_signature(&sig), true); - assert_eq!(status_cache.get_signature_status(&sig), Some(Ok(())),); + assert_eq!(status_cache.get_signature_status(&sig), Some(Ok(()))); } #[test] @@ -190,7 +210,7 @@ mod tests { assert_eq!(first.get_signature_status(&sig), Some(Ok(()))); let last_id = hash(last_id.as_ref()); first.new_cache(&last_id); - assert_eq!(first.get_signature_status(&sig), Some(Ok(())),); + assert_eq!(first.get_signature_status(&sig), Some(Ok(()))); assert!(first.has_signature(&sig)); first.clear(); assert_eq!(first.get_signature_status(&sig), None); @@ -213,31 +233,58 @@ mod tests { } #[test] - fn test_has_signature_merged1() { + fn test_status_cache_merge_parents_has_signature() { let sig = Signature::default(); let last_id = hash(Hash::default().as_ref()); let mut first = BankStatusCache::new(&last_id); first.add(&sig); assert_eq!(first.get_signature_status(&sig), Some(Ok(()))); + + // give first a merge let last_id = hash(last_id.as_ref()); - let second = BankStatusCache::new(&last_id); - first.merge_into_root(second); - assert_eq!(first.get_signature_status(&sig), Some(Ok(())),); - assert!(first.has_signature(&sig)); + first.new_cache(&last_id); + + let last_id = hash(last_id.as_ref()); + let mut second = BankStatusCache::new(&last_id); + + second.merge_parents(&[&first]); + + assert_eq!(second.get_signature_status(&sig), Some(Ok(()))); + assert!(second.has_signature(&sig)); } #[test] - fn test_has_signature_merged2() { + #[ignore] // takes a lot of time or RAM or both.. + fn test_status_cache_merge_parents_overflow() { + let mut last_id = hash(Hash::default().as_ref()); + let mut cache = BankStatusCache::new(&last_id); + + let parents: Vec<_> = (0..MAX_CACHE_ENTRIES) + .map(|_| { + last_id = hash(last_id.as_ref()); + + BankStatusCache::new(&last_id) + }) + .collect(); + + let mut parents_refs: Vec<_> = parents.iter().collect(); + + last_id = hash(Hash::default().as_ref()); + let mut root = BankStatusCache::new(&last_id); + let sig = Signature::default(); - let last_id = hash(Hash::default().as_ref()); - let mut first = BankStatusCache::new(&last_id); - first.add(&sig); - assert_eq!(first.get_signature_status(&sig), Some(Ok(()))); - let last_id = hash(last_id.as_ref()); - let mut second = BankStatusCache::new(&last_id); - second.merge_into_root(first); - assert_eq!(second.get_signature_status(&sig), Some(Ok(())),); - assert!(second.has_signature(&sig)); + root.add(&sig); + + parents_refs.push(&root); + + assert_eq!(root.get_signature_status(&sig), Some(Ok(()))); + assert!(root.has_signature(&sig)); + + // will overflow + cache.merge_parents(&parents_refs); + + assert_eq!(cache.get_signature_status(&sig), None); + assert!(!cache.has_signature(&sig)); } #[test] @@ -268,7 +315,7 @@ mod tests { ); first.clear(); assert_eq!(first.has_signature(&sig), false); - assert_eq!(first.get_signature_status(&sig), None,); + assert_eq!(first.get_signature_status(&sig), None); } #[test] fn test_clear_signatures_all() {