diff --git a/runtime/src/hybrid_btree_map.rs b/runtime/src/hybrid_btree_map.rs deleted file mode 100644 index 785833373e..0000000000 --- a/runtime/src/hybrid_btree_map.rs +++ /dev/null @@ -1,458 +0,0 @@ -use crate::accounts_index::AccountMapEntry; -use crate::accounts_index::{IsCached, RefCount, SlotList, ACCOUNTS_INDEX_CONFIG_FOR_TESTING}; -use crate::bucket_map_holder::{BucketMapWriteHolder}; - -use crate::pubkey_bins::PubkeyBinCalculator16; -use solana_bucket_map::bucket_map::BucketMap; -use solana_sdk::clock::Slot; -use solana_sdk::pubkey::Pubkey; -use std::collections::btree_map::BTreeMap; -use std::fmt::Debug; -use std::ops::Bound; -use std::ops::{Range, RangeBounds}; -use std::sync::Arc; - -type K = Pubkey; - -#[derive(Clone, Debug)] -pub struct HybridAccountEntry { - entry: V, - //exists_on_disk: bool, -} -//type V2 = HybridAccountEntry; -pub type V2 = AccountMapEntry; -/* -trait RealEntry { - fn real_entry(&self) -> T; -} - -impl RealEntry for T { - fn real_entry(&self) -> T - { - self - } -} -*/ -pub type SlotT = (Slot, T); - -#[derive(Debug)] -pub struct HybridBTreeMap { - in_memory: BTreeMap>, - disk: Arc>, - bin_index: usize, - bins: usize, -} - -// TODO: we need a bit for 'exists on disk' for updates -/* -impl Default for HybridBTreeMap { - /// Creates an empty `BTreeMap`. - fn default() -> HybridBTreeMap { - Self { - in_memory: BTreeMap::default(), - disk: BucketMap::new_buckets(PubkeyBinCalculator16::log_2(BINS as u32) as u8), - } - } -} -*/ - -/* -impl<'a, K: 'a, V: 'a> Iterator for HybridBTreeMap<'a, V> { - type Item = (&'a K, &'a V); - - fn next(&mut self) -> Option<(&'a K, &'a V)> { - if self.length == 0 { - None - } else { - self.length -= 1; - Some(unsafe { self.range.inner.next_unchecked() }) - } - } - - fn size_hint(&self) -> (usize, Option) { - (self.length, Some(self.length)) - } - - fn last(mut self) -> Option<(&'a K, &'a V)> { - self.next_back() - } - - fn min(mut self) -> Option<(&'a K, &'a V)> { - self.next() - } - - fn max(mut self) -> Option<(&'a K, &'a V)> { - self.next_back() - } -} -*/ - -pub enum HybridEntry<'a, V: 'static + Clone + IsCached + Debug> { - /// A vacant entry. - Vacant(HybridVacantEntry<'a, V>), - - /// An occupied entry. - Occupied(HybridOccupiedEntry<'a, V>), -} - -pub struct Keys { - keys: Vec, - index: usize, -} - -impl Keys { - pub fn len(&self) -> usize { - self.keys.len() - } -} - -impl Iterator for Keys { - type Item = Pubkey; - fn next(&mut self) -> Option { - if self.index >= self.keys.len() { - None - } else { - let r = Some(self.keys[self.index]); - self.index += 1; - r - } - } -} - -pub struct Values { - values: Vec>, - index: usize, -} - -impl Iterator for Values { - type Item = V2; - fn next(&mut self) -> Option { - if self.index >= self.values.len() { - None - } else { - let r = Some(AccountMapEntry { - slot_list: self.values[self.index].clone(), - ref_count: RefCount::MAX, // todo: no clone - }); - self.index += 1; - r - } - } -} - -pub struct HybridOccupiedEntry<'a, V: 'static + Clone + IsCached + Debug> { - pubkey: Pubkey, - entry: V2, - map: &'a HybridBTreeMap, -} -pub struct HybridVacantEntry<'a, V: 'static + Clone + IsCached + Debug> { - pubkey: Pubkey, - map: &'a HybridBTreeMap, -} - -impl<'a, V: 'a + Clone + IsCached + Debug> HybridOccupiedEntry<'a, V> { - pub fn get(&self) -> &V2 { - &self.entry - } - pub fn update(&mut self, new_data: &SlotList, new_rc: Option) { - //error!("update: {}", self.pubkey); - self.map.disk.update( - &self.pubkey, - |previous| { - if previous.is_some() { - //error!("update {} to {:?}", self.pubkey, new_data); - } - Some((new_data.clone(), new_rc.unwrap_or(self.entry.ref_count))) - // TODO no clone here - }, - Some(&self.entry), - ); - let g = self.map.disk.get(&self.pubkey).unwrap(); - assert_eq!(format!("{:?}", g.1), format!("{:?}", new_data)); - } - pub fn addref(&mut self) { - self.entry.ref_count += 1; - - self.map - .disk - .addref(&self.pubkey, self.entry.ref_count, &self.entry.slot_list); - //error!("addref: {}, {}, {:?}", self.pubkey, self.entry.ref_count(), result); - } - pub fn unref(&mut self) { - self.entry.ref_count -= 1; - self.map - .disk - .unref(&self.pubkey, self.entry.ref_count, &self.entry.slot_list); - //error!("addref: {}, {}, {:?}", self.pubkey, self.entry.ref_count(), result); - } - /* - pub fn get_mut(&mut self) -> &mut V2 { - self.entry.get_mut() - } - */ - pub fn key(&self) -> &K { - &self.pubkey - } - pub fn remove(self) { - self.map.disk.delete_key(&self.pubkey) - } -} - -impl<'a, V: 'a + Clone + Debug + IsCached> HybridVacantEntry<'a, V> { - pub fn insert(self, value: V2) { - // -> &'a mut V2 { - /* - let value = V2:: { - entry: value, - //exists_on_disk: false, - }; - */ - //let mut sl = SlotList::default(); - //std::mem::swap(&mut sl, &mut value.slot_list); - self.map.disk.update( - &self.pubkey, - |_previous| { - Some((value.slot_list.clone() /* todo bad */, value.ref_count)) - }, - None, - ); - } -} - -impl HybridBTreeMap { - /// Creates an empty `BTreeMap`. - pub fn new2(bucket_map: &Arc>, bin_index: usize, bins: usize) -> Self { - Self { - in_memory: BTreeMap::default(), - disk: bucket_map.clone(), - bin_index, - bins: bins, //bucket_map.num_buckets(), - } - } - - pub fn new_for_testing() -> Self { - let map = Self::new_bucket_map(ACCOUNTS_INDEX_CONFIG_FOR_TESTING); - Self::new2(&map, 0, 1) - } - - pub fn new_bucket_map(bins: usize) -> Arc> { - let buckets = PubkeyBinCalculator16::log_2(bins as u32) as u8; // make more buckets to try to spread things out - // 15 hopefully avoids too many files open problem - //buckets = std::cmp::min(buckets + 11, 15); // max # that works with open file handles and such - //buckets = - //error!("creating: {} for {}", buckets, BUCKET_BINS); - Arc::new(BucketMapWriteHolder::new(BucketMap::new_buckets(buckets))) - } - - pub fn flush(&self) -> usize { - let num_buckets = self.disk.num_buckets(); - let mystart = num_buckets * self.bin_index / self.bins; - let myend = num_buckets * (self.bin_index + 1) / self.bins; - assert_eq!(myend - mystart, 1, "{}", self.bin_index); - (mystart..myend) - .map(|ix| self.disk.flush(ix, false, None).1) - .sum() - - /* - { - // put entire contents of this map into the disk backing - let mut keys = Vec::with_capacity(self.in_memory.len()); - for k in self.in_memory.keys() { - keys.push(k); - } - self.disk.update_batch(&keys[..], |previous, key, orig_i| { - let item = self.in_memory.get(key); - item.map(|item| (item.slot_list.clone(), item.ref_count())) - }); - self.in_memory.clear(); - }*/ - } - pub fn distribution(&self) { - self.disk.distribution(); - } - fn bound<'a, T>(bound: Bound<&'a T>, unbounded: &'a T) -> &'a T { - match bound { - Bound::Included(b) | Bound::Excluded(b) => b, - _ => unbounded, - } - } - pub fn range(&self, range: Option) -> Vec<(Pubkey, SlotList)> - where - R: RangeBounds, - { - //self.disk.range.fetch_add(1, Ordering::Relaxed); - - let num_buckets = self.disk.num_buckets(); - if self.bin_index != 0 && self.disk.unified_backing { - return vec![]; - } - let mut start = 0; - let mut end = num_buckets; - if let Some(range) = &range { - let default = Pubkey::default(); - let max = Pubkey::new(&[0xff; 32]); - let start_bound = Self::bound(range.start_bound(), &default); - start = self.disk.bucket_ix(start_bound); - // end is exclusive, so it is end + 1 we care about here - let end_bound = Self::bound(range.end_bound(), &max); - end = std::cmp::min(num_buckets, 1 + self.disk.bucket_ix(end_bound)); // ugly - assert!( - start_bound <= end_bound, - "range start is greater than range end" - ); - } - let len = (start..end) - .into_iter() - .map(|ix| self.disk.bucket_len(ix) as usize) - .sum::(); - - let mystart = num_buckets * self.bin_index / self.bins; - let myend = num_buckets * (self.bin_index + 1) / self.bins; - start = std::cmp::max(start, mystart); - end = std::cmp::min(end, myend); - let mut keys = Vec::with_capacity(len); - (start..end).into_iter().for_each(|ix| { - let mut ks = self.disk.range(ix, range.as_ref()); - keys.append(&mut ks); - }); - keys.sort_unstable_by(|a, b| a.0.cmp(&b.0)); - keys - } - - pub fn keys2(&self) -> Keys { - // used still? - let num_buckets = self.disk.num_buckets(); - let start = num_buckets * self.bin_index / self.bins; - let end = num_buckets * (self.bin_index + 1) / self.bins; - let len = (start..end) - .into_iter() - .map(|ix| self.disk.bucket_len(ix) as usize) - .sum::(); - let mut keys = Vec::with_capacity(len); - let _len = (start..end).into_iter().for_each(|ix| { - keys.append( - &mut self - .disk - .keys3(ix, None::<&Range>) - .unwrap_or_default(), - ) - }); - keys.sort_unstable(); - Keys { keys, index: 0 } - } - pub fn values(&self) -> Values { - let num_buckets = self.disk.num_buckets(); - if self.bin_index != 0 && self.disk.unified_backing { - return Values { - values: vec![], - index: 0, - }; - } - // todo: this may be unsafe if we are asking for things with an update cache active. thankfully, we only call values at startup right now - let start = num_buckets * self.bin_index / self.bins; - let end = num_buckets * (self.bin_index + 1) / self.bins; - let len = (start..end) - .into_iter() - .map(|ix| self.disk.bucket_len(ix) as usize) - .sum::(); - let mut values = Vec::with_capacity(len); - (start..end).into_iter().for_each(|ix| { - values.append( - &mut self - .disk - .values(ix, None::<&Range>) - .unwrap_or_default(), - ) - }); - //error!("getting values: {}, bin: {}, bins: {}, start: {}, end: {}", values.len(), self.bin_index, self.bins, start, end); - //keys.sort_unstable(); - if self.bin_index == 0 { - //error!("getting values: {}, {}, {}", values.len(), start, end); - } - Values { values, index: 0 } - } - - pub fn upsert( - &self, - pubkey: &Pubkey, - new_value: AccountMapEntry, - reclaims: &mut SlotList, - reclaims_must_be_empty: bool, - ) { - self.disk - .upsert(pubkey, new_value, reclaims, reclaims_must_be_empty); - } - - pub fn entry(&mut self, key: K) -> HybridEntry<'_, V> { - match self.disk.get(&key) { - Some(entry) => HybridEntry::Occupied(HybridOccupiedEntry { - pubkey: key, - entry: AccountMapEntry:: { - slot_list: entry.1, - ref_count: entry.0, - }, - map: self, - }), - None => HybridEntry::Vacant(HybridVacantEntry { - pubkey: key, - map: self, - }), - } - } - - pub fn insert2(&mut self, key: K, value: V2) { - match self.entry(key) { - HybridEntry::Occupied(_occupied) => { - panic!(""); - } - HybridEntry::Vacant(vacant) => vacant.insert(value), - } - } - - pub fn get(&self, key: &K) -> Option> { - let lookup = || { - let disk = self.disk.get(key); - disk.map(|disk| AccountMapEntry { - ref_count: disk.0, - slot_list: disk.1, - }) - }; - - if true { - lookup() - } else { - let in_mem = self.in_memory.get(key); - match in_mem { - Some(in_mem) => Some(in_mem.clone()), - None => { - // we have to load this into the in-mem cache so we can get a ref_count, if nothing else - lookup() - /* - disk.map(|item| { - self.in_memory.entry(*key).map(|entry| { - - } - })*/ - } - } - } - } - pub fn remove(&mut self, key: &K) { - self.disk.delete_key(key); //.map(|x| x.entry) - } - pub fn len(&self) -> usize { - self.disk.keys3(self.bin_index, None::<&Range>).map(|x| x.len()).unwrap_or_default() - } - - pub fn set_startup(&self, startup: bool) { - self.disk.set_startup(startup); - } - - pub fn update_or_insert_async(&self, pubkey: Pubkey, new_entry: AccountMapEntry) { - self.disk - .update_or_insert_async(self.bin_index, pubkey, new_entry); - } - pub fn dump_metrics(&self) { - self.disk.dump_metrics(); - } -}