Remove bloat from secondary indexes (#17048)
This commit is contained in:
@@ -492,7 +492,7 @@ pub trait ZeroLamport {
|
||||
fn is_zero_lamport(&self) -> bool;
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
#[derive(Debug)]
|
||||
pub struct AccountsIndex<T> {
|
||||
pub account_maps: RwLock<AccountMap<Pubkey, AccountMapEntry<T>>>,
|
||||
program_id_index: SecondaryIndex<DashMapSecondaryIndexEntry>,
|
||||
@@ -503,6 +503,26 @@ pub struct AccountsIndex<T> {
|
||||
zero_lamport_pubkeys: DashSet<Pubkey>,
|
||||
}
|
||||
|
||||
impl<T> Default for AccountsIndex<T> {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
account_maps: RwLock::<AccountMap<Pubkey, AccountMapEntry<T>>>::default(),
|
||||
program_id_index: SecondaryIndex::<DashMapSecondaryIndexEntry>::new(
|
||||
"program_id_index_stats",
|
||||
),
|
||||
spl_token_mint_index: SecondaryIndex::<DashMapSecondaryIndexEntry>::new(
|
||||
"spl_token_mint_index_stats",
|
||||
),
|
||||
spl_token_owner_index: SecondaryIndex::<RwLockSecondaryIndexEntry>::new(
|
||||
"spl_token_owner_index_stats",
|
||||
),
|
||||
roots_tracker: RwLock::<RootsTracker>::default(),
|
||||
ongoing_scan_roots: RwLock::<BTreeMap<Slot, u64>>::default(),
|
||||
zero_lamport_pubkeys: DashSet::<Pubkey>::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
|
||||
fn iter<R>(&self, range: Option<R>) -> AccountsIndexIterator<T>
|
||||
where
|
||||
@@ -858,15 +878,10 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
|
||||
if index_entry.get().slot_list.read().unwrap().is_empty() {
|
||||
index_entry.remove();
|
||||
|
||||
// Note passing `None` to remove all the entries for this key
|
||||
// is only safe because we have the lock for this key's entry
|
||||
// in the AccountsIndex, so no other thread is also updating
|
||||
// the index
|
||||
self.purge_secondary_indexes_by_inner_key(
|
||||
key,
|
||||
None::<&Slot>,
|
||||
account_indexes,
|
||||
);
|
||||
// Note it's only safe to remove all the entries for this key
|
||||
// because we have the lock for this key's entry in the AccountsIndex,
|
||||
// so no other thread is also updating the index
|
||||
self.purge_secondary_indexes_by_inner_key(key, account_indexes);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -954,28 +969,23 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
|
||||
pubkey: &Pubkey,
|
||||
slots_to_purge: &'a C,
|
||||
reclaims: &mut SlotList<T>,
|
||||
account_indexes: &AccountSecondaryIndexes,
|
||||
) -> bool
|
||||
where
|
||||
C: Contains<'a, Slot>,
|
||||
{
|
||||
let is_empty = {
|
||||
let mut write_account_map_entry = self.get_account_write_entry(pubkey).unwrap();
|
||||
write_account_map_entry.slot_list_mut(|slot_list| {
|
||||
slot_list.retain(|(slot, item)| {
|
||||
let should_purge = slots_to_purge.contains(&slot);
|
||||
if should_purge {
|
||||
reclaims.push((*slot, item.clone()));
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
slot_list.is_empty()
|
||||
})
|
||||
};
|
||||
self.purge_secondary_indexes_by_inner_key(pubkey, Some(slots_to_purge), account_indexes);
|
||||
is_empty
|
||||
let mut write_account_map_entry = self.get_account_write_entry(pubkey).unwrap();
|
||||
write_account_map_entry.slot_list_mut(|slot_list| {
|
||||
slot_list.retain(|(slot, item)| {
|
||||
let should_purge = slots_to_purge.contains(&slot);
|
||||
if should_purge {
|
||||
reclaims.push((*slot, item.clone()));
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
slot_list.is_empty()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn min_ongoing_scan_root(&self) -> Option<Slot> {
|
||||
@@ -1079,7 +1089,6 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
|
||||
fn update_secondary_indexes(
|
||||
&self,
|
||||
pubkey: &Pubkey,
|
||||
slot: Slot,
|
||||
account_owner: &Pubkey,
|
||||
account_data: &[u8],
|
||||
account_indexes: &AccountSecondaryIndexes,
|
||||
@@ -1091,7 +1100,7 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
|
||||
if account_indexes.contains(&AccountIndex::ProgramId)
|
||||
&& account_indexes.include_key(account_owner)
|
||||
{
|
||||
self.program_id_index.insert(account_owner, pubkey, slot);
|
||||
self.program_id_index.insert(account_owner, pubkey);
|
||||
}
|
||||
// Note because of the below check below on the account data length, when an
|
||||
// account hits zero lamports and is reset to AccountSharedData::Default, then we skip
|
||||
@@ -1115,7 +1124,7 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
|
||||
..SPL_TOKEN_ACCOUNT_OWNER_OFFSET + PUBKEY_BYTES],
|
||||
);
|
||||
if account_indexes.include_key(&owner_key) {
|
||||
self.spl_token_owner_index.insert(&owner_key, pubkey, slot);
|
||||
self.spl_token_owner_index.insert(&owner_key, pubkey);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1125,7 +1134,7 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
|
||||
..SPL_TOKEN_ACCOUNT_MINT_OFFSET + PUBKEY_BYTES],
|
||||
);
|
||||
if account_indexes.include_key(&mint_key) {
|
||||
self.spl_token_mint_index.insert(&mint_key, pubkey, slot);
|
||||
self.spl_token_mint_index.insert(&mint_key, pubkey);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1151,7 +1160,7 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
|
||||
}
|
||||
w_account_entry.update(slot, account_info, reclaims);
|
||||
}
|
||||
self.update_secondary_indexes(pubkey, slot, account_owner, account_data, account_indexes);
|
||||
self.update_secondary_indexes(pubkey, account_owner, account_data, account_indexes);
|
||||
}
|
||||
|
||||
// Updates the given pubkey at the given slot with the new account information.
|
||||
@@ -1187,7 +1196,7 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
|
||||
w_account_entry.update(slot, account_info, reclaims);
|
||||
is_newly_inserted
|
||||
};
|
||||
self.update_secondary_indexes(pubkey, slot, account_owner, account_data, account_indexes);
|
||||
self.update_secondary_indexes(pubkey, account_owner, account_data, account_indexes);
|
||||
is_newly_inserted
|
||||
}
|
||||
|
||||
@@ -1213,37 +1222,29 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
|
||||
}
|
||||
}
|
||||
|
||||
fn purge_secondary_indexes_by_inner_key<'a, C>(
|
||||
fn purge_secondary_indexes_by_inner_key<'a>(
|
||||
&'a self,
|
||||
inner_key: &Pubkey,
|
||||
slots_to_remove: Option<&'a C>,
|
||||
account_indexes: &AccountSecondaryIndexes,
|
||||
) where
|
||||
C: Contains<'a, Slot>,
|
||||
{
|
||||
) {
|
||||
if account_indexes.contains(&AccountIndex::ProgramId) {
|
||||
self.program_id_index
|
||||
.remove_by_inner_key(inner_key, slots_to_remove);
|
||||
self.program_id_index.remove_by_inner_key(inner_key);
|
||||
}
|
||||
|
||||
if account_indexes.contains(&AccountIndex::SplTokenOwner) {
|
||||
self.spl_token_owner_index
|
||||
.remove_by_inner_key(inner_key, slots_to_remove);
|
||||
self.spl_token_owner_index.remove_by_inner_key(inner_key);
|
||||
}
|
||||
|
||||
if account_indexes.contains(&AccountIndex::SplTokenMint) {
|
||||
self.spl_token_mint_index
|
||||
.remove_by_inner_key(inner_key, slots_to_remove);
|
||||
self.spl_token_mint_index.remove_by_inner_key(inner_key);
|
||||
}
|
||||
}
|
||||
|
||||
fn purge_older_root_entries(
|
||||
&self,
|
||||
pubkey: &Pubkey,
|
||||
slot_list: &mut SlotList<T>,
|
||||
reclaims: &mut SlotList<T>,
|
||||
max_clean_root: Option<Slot>,
|
||||
account_indexes: &AccountSecondaryIndexes,
|
||||
) {
|
||||
let roots_tracker = &self.roots_tracker.read().unwrap();
|
||||
let newest_root_in_slot_list =
|
||||
@@ -1261,8 +1262,6 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
|
||||
}
|
||||
!should_purge
|
||||
});
|
||||
|
||||
self.purge_secondary_indexes_by_inner_key(pubkey, Some(&purged_slots), account_indexes);
|
||||
}
|
||||
|
||||
pub fn clean_rooted_entries(
|
||||
@@ -1270,18 +1269,11 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
|
||||
pubkey: &Pubkey,
|
||||
reclaims: &mut SlotList<T>,
|
||||
max_clean_root: Option<Slot>,
|
||||
account_indexes: &AccountSecondaryIndexes,
|
||||
) {
|
||||
let mut is_slot_list_empty = false;
|
||||
if let Some(mut locked_entry) = self.get_account_write_entry(pubkey) {
|
||||
locked_entry.slot_list_mut(|slot_list| {
|
||||
self.purge_older_root_entries(
|
||||
pubkey,
|
||||
slot_list,
|
||||
reclaims,
|
||||
max_clean_root,
|
||||
account_indexes,
|
||||
);
|
||||
self.purge_older_root_entries(slot_list, reclaims, max_clean_root);
|
||||
is_slot_list_empty = slot_list.is_empty();
|
||||
});
|
||||
}
|
||||
@@ -2755,7 +2747,7 @@ pub mod tests {
|
||||
secondary_index: &SecondaryIndex<SecondaryIndexEntryType>,
|
||||
key_start: usize,
|
||||
key_end: usize,
|
||||
account_index: &AccountSecondaryIndexes,
|
||||
secondary_indexes: &AccountSecondaryIndexes,
|
||||
) {
|
||||
// No roots, should be no reclaims
|
||||
let slots = vec![1, 2, 5, 9];
|
||||
@@ -2773,7 +2765,7 @@ pub mod tests {
|
||||
// Make sure these accounts are added to secondary index
|
||||
&inline_spl_token_v2_0::id(),
|
||||
&account_data,
|
||||
account_index,
|
||||
secondary_indexes,
|
||||
true,
|
||||
&mut vec![],
|
||||
);
|
||||
@@ -2793,43 +2785,43 @@ pub mod tests {
|
||||
.read()
|
||||
.unwrap()
|
||||
.len(),
|
||||
slots.len()
|
||||
1
|
||||
);
|
||||
|
||||
index.purge_exact(
|
||||
&account_key,
|
||||
&slots.into_iter().collect::<HashSet<Slot>>(),
|
||||
&mut vec![],
|
||||
account_index,
|
||||
);
|
||||
|
||||
index.handle_dead_keys(&[&account_key], secondary_indexes);
|
||||
assert!(secondary_index.index.is_empty());
|
||||
assert!(secondary_index.reverse_index.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_purge_exact_dashmap_secondary_index() {
|
||||
let (key_start, key_end, account_index) = create_dashmap_secondary_index_state();
|
||||
let (key_start, key_end, secondary_indexes) = create_dashmap_secondary_index_state();
|
||||
let index = AccountsIndex::<bool>::default();
|
||||
run_test_purge_exact_secondary_index(
|
||||
&index,
|
||||
&index.spl_token_mint_index,
|
||||
key_start,
|
||||
key_end,
|
||||
&account_index,
|
||||
&secondary_indexes,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_purge_exact_rwlock_secondary_index() {
|
||||
let (key_start, key_end, account_index) = create_rwlock_secondary_index_state();
|
||||
let (key_start, key_end, secondary_indexes) = create_rwlock_secondary_index_state();
|
||||
let index = AccountsIndex::<bool>::default();
|
||||
run_test_purge_exact_secondary_index(
|
||||
&index,
|
||||
&index.spl_token_owner_index,
|
||||
key_start,
|
||||
key_end,
|
||||
&account_index,
|
||||
&secondary_indexes,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2839,13 +2831,7 @@ pub mod tests {
|
||||
let index = AccountsIndex::<bool>::default();
|
||||
let mut slot_list = vec![(1, true), (2, true), (5, true), (9, true)];
|
||||
let mut reclaims = vec![];
|
||||
index.purge_older_root_entries(
|
||||
&Pubkey::default(),
|
||||
&mut slot_list,
|
||||
&mut reclaims,
|
||||
None,
|
||||
&AccountSecondaryIndexes::default(),
|
||||
);
|
||||
index.purge_older_root_entries(&mut slot_list, &mut reclaims, None);
|
||||
assert!(reclaims.is_empty());
|
||||
assert_eq!(slot_list, vec![(1, true), (2, true), (5, true), (9, true)]);
|
||||
|
||||
@@ -2855,13 +2841,7 @@ pub mod tests {
|
||||
// Note 2 is not a root
|
||||
index.add_root(5, false);
|
||||
reclaims = vec![];
|
||||
index.purge_older_root_entries(
|
||||
&Pubkey::default(),
|
||||
&mut slot_list,
|
||||
&mut reclaims,
|
||||
None,
|
||||
&AccountSecondaryIndexes::default(),
|
||||
);
|
||||
index.purge_older_root_entries(&mut slot_list, &mut reclaims, None);
|
||||
assert_eq!(reclaims, vec![(1, true), (2, true)]);
|
||||
assert_eq!(slot_list, vec![(5, true), (9, true)]);
|
||||
|
||||
@@ -2869,13 +2849,7 @@ pub mod tests {
|
||||
slot_list = vec![(1, true), (2, true), (5, true), (9, true)];
|
||||
index.add_root(6, false);
|
||||
reclaims = vec![];
|
||||
index.purge_older_root_entries(
|
||||
&Pubkey::default(),
|
||||
&mut slot_list,
|
||||
&mut reclaims,
|
||||
None,
|
||||
&AccountSecondaryIndexes::default(),
|
||||
);
|
||||
index.purge_older_root_entries(&mut slot_list, &mut reclaims, None);
|
||||
assert_eq!(reclaims, vec![(1, true), (2, true)]);
|
||||
assert_eq!(slot_list, vec![(5, true), (9, true)]);
|
||||
|
||||
@@ -2883,26 +2857,14 @@ pub mod tests {
|
||||
// outcome
|
||||
slot_list = vec![(1, true), (2, true), (5, true), (9, true)];
|
||||
reclaims = vec![];
|
||||
index.purge_older_root_entries(
|
||||
&Pubkey::default(),
|
||||
&mut slot_list,
|
||||
&mut reclaims,
|
||||
Some(6),
|
||||
&AccountSecondaryIndexes::default(),
|
||||
);
|
||||
index.purge_older_root_entries(&mut slot_list, &mut reclaims, Some(6));
|
||||
assert_eq!(reclaims, vec![(1, true), (2, true)]);
|
||||
assert_eq!(slot_list, vec![(5, true), (9, true)]);
|
||||
|
||||
// Pass a max root, earlier slots should be reclaimed
|
||||
slot_list = vec![(1, true), (2, true), (5, true), (9, true)];
|
||||
reclaims = vec![];
|
||||
index.purge_older_root_entries(
|
||||
&Pubkey::default(),
|
||||
&mut slot_list,
|
||||
&mut reclaims,
|
||||
Some(5),
|
||||
&AccountSecondaryIndexes::default(),
|
||||
);
|
||||
index.purge_older_root_entries(&mut slot_list, &mut reclaims, Some(5));
|
||||
assert_eq!(reclaims, vec![(1, true), (2, true)]);
|
||||
assert_eq!(slot_list, vec![(5, true), (9, true)]);
|
||||
|
||||
@@ -2910,13 +2872,7 @@ pub mod tests {
|
||||
// so nothing will be purged
|
||||
slot_list = vec![(1, true), (2, true), (5, true), (9, true)];
|
||||
reclaims = vec![];
|
||||
index.purge_older_root_entries(
|
||||
&Pubkey::default(),
|
||||
&mut slot_list,
|
||||
&mut reclaims,
|
||||
Some(2),
|
||||
&AccountSecondaryIndexes::default(),
|
||||
);
|
||||
index.purge_older_root_entries(&mut slot_list, &mut reclaims, Some(2));
|
||||
assert!(reclaims.is_empty());
|
||||
assert_eq!(slot_list, vec![(1, true), (2, true), (5, true), (9, true)]);
|
||||
|
||||
@@ -2924,13 +2880,7 @@ pub mod tests {
|
||||
// so nothing will be purged
|
||||
slot_list = vec![(1, true), (2, true), (5, true), (9, true)];
|
||||
reclaims = vec![];
|
||||
index.purge_older_root_entries(
|
||||
&Pubkey::default(),
|
||||
&mut slot_list,
|
||||
&mut reclaims,
|
||||
Some(1),
|
||||
&AccountSecondaryIndexes::default(),
|
||||
);
|
||||
index.purge_older_root_entries(&mut slot_list, &mut reclaims, Some(1));
|
||||
assert!(reclaims.is_empty());
|
||||
assert_eq!(slot_list, vec![(1, true), (2, true), (5, true), (9, true)]);
|
||||
|
||||
@@ -2938,41 +2888,32 @@ pub mod tests {
|
||||
// some of the roots in the list, shouldn't return those smaller roots
|
||||
slot_list = vec![(1, true), (2, true), (5, true), (9, true)];
|
||||
reclaims = vec![];
|
||||
index.purge_older_root_entries(
|
||||
&Pubkey::default(),
|
||||
&mut slot_list,
|
||||
&mut reclaims,
|
||||
Some(7),
|
||||
&AccountSecondaryIndexes::default(),
|
||||
);
|
||||
index.purge_older_root_entries(&mut slot_list, &mut reclaims, Some(7));
|
||||
assert_eq!(reclaims, vec![(1, true), (2, true)]);
|
||||
assert_eq!(slot_list, vec![(5, true), (9, true)]);
|
||||
}
|
||||
|
||||
fn check_secondary_index_unique<SecondaryIndexEntryType>(
|
||||
fn check_secondary_index_mapping_correct<SecondaryIndexEntryType>(
|
||||
secondary_index: &SecondaryIndex<SecondaryIndexEntryType>,
|
||||
slot: Slot,
|
||||
key: &Pubkey,
|
||||
secondary_index_keys: &[Pubkey],
|
||||
account_key: &Pubkey,
|
||||
) where
|
||||
SecondaryIndexEntryType: SecondaryIndexEntry + Default + Sync + Send,
|
||||
{
|
||||
// Check secondary index has unique mapping from secondary index key
|
||||
// to the account key and slot
|
||||
assert_eq!(secondary_index.index.len(), 1);
|
||||
let inner_key_map = secondary_index.index.get(key).unwrap();
|
||||
assert_eq!(inner_key_map.len(), 1);
|
||||
inner_key_map
|
||||
.value()
|
||||
.get(account_key, &|slots_map: Option<&RwLock<HashSet<Slot>>>| {
|
||||
let slots_map = slots_map.unwrap();
|
||||
assert_eq!(slots_map.read().unwrap().len(), 1);
|
||||
assert!(slots_map.read().unwrap().contains(&slot));
|
||||
});
|
||||
|
||||
// Check reverse index is unique
|
||||
let slots_map = secondary_index.reverse_index.get(account_key).unwrap();
|
||||
assert_eq!(slots_map.value().read().unwrap().get(&slot).unwrap(), key);
|
||||
for secondary_index_key in secondary_index_keys {
|
||||
assert_eq!(secondary_index.index.len(), secondary_index_keys.len());
|
||||
let account_key_map = secondary_index.get(secondary_index_key);
|
||||
assert_eq!(account_key_map.len(), 1);
|
||||
assert_eq!(account_key_map, vec![*account_key]);
|
||||
}
|
||||
// Check reverse index contains all of the `secondary_index_keys`
|
||||
let secondary_index_key_map = secondary_index.reverse_index.get(account_key).unwrap();
|
||||
assert_eq!(
|
||||
&*secondary_index_key_map.value().read().unwrap(),
|
||||
secondary_index_keys
|
||||
);
|
||||
}
|
||||
|
||||
fn run_test_secondary_indexes<
|
||||
@@ -2982,12 +2923,11 @@ pub mod tests {
|
||||
secondary_index: &SecondaryIndex<SecondaryIndexEntryType>,
|
||||
key_start: usize,
|
||||
key_end: usize,
|
||||
account_index: &AccountSecondaryIndexes,
|
||||
secondary_indexes: &AccountSecondaryIndexes,
|
||||
) {
|
||||
let mut account_index = account_index.clone();
|
||||
let mut secondary_indexes = secondary_indexes.clone();
|
||||
let account_key = Pubkey::new_unique();
|
||||
let index_key = Pubkey::new_unique();
|
||||
let slot = 1;
|
||||
let mut account_data = vec![0; inline_spl_token_v2_0::state::Account::get_packed_len()];
|
||||
account_data[key_start..key_end].clone_from_slice(&(index_key.to_bytes()));
|
||||
|
||||
@@ -2997,7 +2937,7 @@ pub mod tests {
|
||||
&account_key,
|
||||
&Pubkey::default(),
|
||||
&account_data,
|
||||
&account_index,
|
||||
&secondary_indexes,
|
||||
true,
|
||||
&mut vec![],
|
||||
);
|
||||
@@ -3010,66 +2950,31 @@ pub mod tests {
|
||||
&account_key,
|
||||
&inline_spl_token_v2_0::id(),
|
||||
&account_data[1..],
|
||||
&account_index,
|
||||
&secondary_indexes,
|
||||
true,
|
||||
&mut vec![],
|
||||
);
|
||||
assert!(secondary_index.index.is_empty());
|
||||
assert!(secondary_index.reverse_index.is_empty());
|
||||
|
||||
// excluded key
|
||||
account_index.keys = Some(AccountSecondaryIndexesIncludeExclude {
|
||||
keys: [index_key].iter().cloned().collect::<HashSet<_>>(),
|
||||
exclude: true,
|
||||
});
|
||||
index.upsert(
|
||||
0,
|
||||
&account_key,
|
||||
&inline_spl_token_v2_0::id(),
|
||||
&account_data[1..],
|
||||
&account_index,
|
||||
true,
|
||||
&mut vec![],
|
||||
);
|
||||
assert!(secondary_index.index.is_empty());
|
||||
assert!(secondary_index.reverse_index.is_empty());
|
||||
|
||||
// not-included key
|
||||
account_index.keys = Some(AccountSecondaryIndexesIncludeExclude {
|
||||
keys: [account_key].iter().cloned().collect::<HashSet<_>>(),
|
||||
exclude: false,
|
||||
});
|
||||
index.upsert(
|
||||
0,
|
||||
&account_key,
|
||||
&inline_spl_token_v2_0::id(),
|
||||
&account_data[1..],
|
||||
&account_index,
|
||||
true,
|
||||
&mut vec![],
|
||||
);
|
||||
assert!(secondary_index.index.is_empty());
|
||||
assert!(secondary_index.reverse_index.is_empty());
|
||||
|
||||
account_index.keys = None;
|
||||
secondary_indexes.keys = None;
|
||||
|
||||
// Just right. Inserting the same index multiple times should be ok
|
||||
for _ in 0..2 {
|
||||
index.update_secondary_indexes(
|
||||
&account_key,
|
||||
slot,
|
||||
&inline_spl_token_v2_0::id(),
|
||||
&account_data,
|
||||
&account_index,
|
||||
&secondary_indexes,
|
||||
);
|
||||
check_secondary_index_unique(secondary_index, slot, &index_key, &account_key);
|
||||
check_secondary_index_mapping_correct(secondary_index, &[index_key], &account_key);
|
||||
}
|
||||
|
||||
// included
|
||||
assert!(!secondary_index.index.is_empty());
|
||||
assert!(!secondary_index.reverse_index.is_empty());
|
||||
|
||||
account_index.keys = Some(AccountSecondaryIndexesIncludeExclude {
|
||||
secondary_indexes.keys = Some(AccountSecondaryIndexesIncludeExclude {
|
||||
keys: [index_key].iter().cloned().collect::<HashSet<_>>(),
|
||||
exclude: false,
|
||||
});
|
||||
@@ -3077,17 +2982,16 @@ pub mod tests {
|
||||
secondary_index.reverse_index.clear();
|
||||
index.update_secondary_indexes(
|
||||
&account_key,
|
||||
slot,
|
||||
&inline_spl_token_v2_0::id(),
|
||||
&account_data,
|
||||
&account_index,
|
||||
&secondary_indexes,
|
||||
);
|
||||
assert!(!secondary_index.index.is_empty());
|
||||
assert!(!secondary_index.reverse_index.is_empty());
|
||||
check_secondary_index_unique(secondary_index, slot, &index_key, &account_key);
|
||||
check_secondary_index_mapping_correct(secondary_index, &[index_key], &account_key);
|
||||
|
||||
// not-excluded
|
||||
account_index.keys = Some(AccountSecondaryIndexesIncludeExclude {
|
||||
secondary_indexes.keys = Some(AccountSecondaryIndexesIncludeExclude {
|
||||
keys: [].iter().cloned().collect::<HashSet<_>>(),
|
||||
exclude: true,
|
||||
});
|
||||
@@ -3095,16 +2999,15 @@ pub mod tests {
|
||||
secondary_index.reverse_index.clear();
|
||||
index.update_secondary_indexes(
|
||||
&account_key,
|
||||
slot,
|
||||
&inline_spl_token_v2_0::id(),
|
||||
&account_data,
|
||||
&account_index,
|
||||
&secondary_indexes,
|
||||
);
|
||||
assert!(!secondary_index.index.is_empty());
|
||||
assert!(!secondary_index.reverse_index.is_empty());
|
||||
check_secondary_index_unique(secondary_index, slot, &index_key, &account_key);
|
||||
check_secondary_index_mapping_correct(secondary_index, &[index_key], &account_key);
|
||||
|
||||
account_index.keys = None;
|
||||
secondary_indexes.keys = None;
|
||||
|
||||
index
|
||||
.get_account_write_entry(&account_key)
|
||||
@@ -3112,34 +3015,34 @@ pub mod tests {
|
||||
.slot_list_mut(|slot_list| slot_list.clear());
|
||||
|
||||
// Everything should be deleted
|
||||
index.handle_dead_keys(&[&account_key], &account_index);
|
||||
index.handle_dead_keys(&[&account_key], &secondary_indexes);
|
||||
assert!(secondary_index.index.is_empty());
|
||||
assert!(secondary_index.reverse_index.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dashmap_secondary_index() {
|
||||
let (key_start, key_end, account_index) = create_dashmap_secondary_index_state();
|
||||
let (key_start, key_end, secondary_indexes) = create_dashmap_secondary_index_state();
|
||||
let index = AccountsIndex::<bool>::default();
|
||||
run_test_secondary_indexes(
|
||||
&index,
|
||||
&index.spl_token_mint_index,
|
||||
key_start,
|
||||
key_end,
|
||||
&account_index,
|
||||
&secondary_indexes,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rwlock_secondary_index() {
|
||||
let (key_start, key_end, account_index) = create_rwlock_secondary_index_state();
|
||||
let (key_start, key_end, secondary_indexes) = create_rwlock_secondary_index_state();
|
||||
let index = AccountsIndex::<bool>::default();
|
||||
run_test_secondary_indexes(
|
||||
&index,
|
||||
&index.spl_token_owner_index,
|
||||
key_start,
|
||||
key_end,
|
||||
&account_index,
|
||||
&secondary_indexes,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -3150,7 +3053,7 @@ pub mod tests {
|
||||
secondary_index: &SecondaryIndex<SecondaryIndexEntryType>,
|
||||
index_key_start: usize,
|
||||
index_key_end: usize,
|
||||
account_index: &AccountSecondaryIndexes,
|
||||
secondary_indexes: &AccountSecondaryIndexes,
|
||||
) {
|
||||
let account_key = Pubkey::new_unique();
|
||||
let secondary_key1 = Pubkey::new_unique();
|
||||
@@ -3169,61 +3072,66 @@ pub mod tests {
|
||||
&account_key,
|
||||
&inline_spl_token_v2_0::id(),
|
||||
&account_data1,
|
||||
account_index,
|
||||
secondary_indexes,
|
||||
true,
|
||||
&mut vec![],
|
||||
);
|
||||
|
||||
// Now write a different mint index
|
||||
// Now write a different mint index for the same account
|
||||
index.upsert(
|
||||
slot,
|
||||
&account_key,
|
||||
&inline_spl_token_v2_0::id(),
|
||||
&account_data2,
|
||||
account_index,
|
||||
secondary_indexes,
|
||||
true,
|
||||
&mut vec![],
|
||||
);
|
||||
|
||||
// Check correctness
|
||||
check_secondary_index_unique(&secondary_index, slot, &secondary_key2, &account_key);
|
||||
assert!(secondary_index.get(&secondary_key1).is_empty());
|
||||
assert_eq!(secondary_index.get(&secondary_key2), vec![account_key]);
|
||||
// Both pubkeys will now be present in the index
|
||||
check_secondary_index_mapping_correct(
|
||||
&secondary_index,
|
||||
&[secondary_key1, secondary_key2],
|
||||
&account_key,
|
||||
);
|
||||
|
||||
// If another fork reintroduces secondary_key1, then it should be re-added to the
|
||||
// index
|
||||
let fork = slot + 1;
|
||||
// If a later slot also introduces secondary_key1, then it should still exist in the index
|
||||
let later_slot = slot + 1;
|
||||
index.upsert(
|
||||
fork,
|
||||
later_slot,
|
||||
&account_key,
|
||||
&inline_spl_token_v2_0::id(),
|
||||
&account_data1,
|
||||
account_index,
|
||||
secondary_indexes,
|
||||
true,
|
||||
&mut vec![],
|
||||
);
|
||||
assert_eq!(secondary_index.get(&secondary_key1), vec![account_key]);
|
||||
|
||||
// If we set a root at fork, and clean, then the secondary_key1 should no longer
|
||||
// be findable
|
||||
index.add_root(fork, false);
|
||||
// If we set a root at `later_slot`, and clean, then even though the account with secondary_key1
|
||||
// was outdated by the update in the later slot, the primary account key is still alive,
|
||||
// so both secondary keys will still be kept alive.
|
||||
index.add_root(later_slot, false);
|
||||
index
|
||||
.get_account_write_entry(&account_key)
|
||||
.unwrap()
|
||||
.slot_list_mut(|slot_list| {
|
||||
index.purge_older_root_entries(
|
||||
&account_key,
|
||||
slot_list,
|
||||
&mut vec![],
|
||||
None,
|
||||
account_index,
|
||||
)
|
||||
index.purge_older_root_entries(slot_list, &mut vec![], None)
|
||||
});
|
||||
assert!(secondary_index.get(&secondary_key2).is_empty());
|
||||
assert_eq!(secondary_index.get(&secondary_key1), vec![account_key]);
|
||||
|
||||
// Check correctness
|
||||
check_secondary_index_unique(secondary_index, fork, &secondary_key1, &account_key);
|
||||
check_secondary_index_mapping_correct(
|
||||
secondary_index,
|
||||
&[secondary_key1, secondary_key2],
|
||||
&account_key,
|
||||
);
|
||||
|
||||
// Removing the remaining entry for this pubkey in the index should mark the
|
||||
// pubkey as dead and finally remove all the secondary indexes
|
||||
let mut reclaims = vec![];
|
||||
index.purge_exact(&account_key, &later_slot, &mut reclaims);
|
||||
index.handle_dead_keys(&[&account_key], secondary_indexes);
|
||||
assert!(secondary_index.index.is_empty());
|
||||
assert!(secondary_index.reverse_index.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
Reference in New Issue
Block a user