introduce bucket map (#19848)

* introduce bucket map

* rename BucketMap bits to num_buckets_pow2

* use u64::BITS

* Store the number of buckets in BucketMapConfig as a regular number

* remove redundant type aliases

* use Slot from sdk

* use Arc::clone() instead

* fixup erase drives

* rename num_buckets to max_buckets

* add doc to BucketMapConfig::new()

* add more documentation

* rename to DEFAULT_CAPACITY_POW2

* doc

* add more traits while we can

* rename capacity to capacity_pow2

* fix a naming for max_buckets_pow2

* remove unused/incorrect DataBucket::bytes

* rework benches a bit

* fixup bench docs

* rename create_bucket_capacity_pow2 to bucket_capacity_when_created_pow2

* rename BucketMapKeyValue to BucketItem

* rename to items_in_range

* remove values()

* remove addref and unref

* remove more addref and unref

* resurect addref and unref since tests use 'em for now

* rename to BucketStorage

* move stats in bucket_stats

* remove specializations (i don't think they are needed)

* move MaxSearch and RefCount into lib.rs

* move BucketItem to bucket_item.rs

* add doc

* keys no longer returns an option

* Revert "remove specializations (i don't think they are needed)"

This reverts commit b22f78e072.

Co-authored-by: Brooks Prumo <brooks@solana.com>

bucket_map/benches/bucket_map.rs

@@ -0,0 +1,76 @@
+#![feature(test)]
+
+macro_rules! DEFINE_NxM_BENCH {
+    ($i:ident, $n:literal, $m:literal) => {
+        mod $i {
+            use super::*;
+
+            #[bench]
+            fn bench_insert_baseline_hashmap(bencher: &mut Bencher) {
+                do_bench_insert_baseline_hashmap(bencher, $n, $m);
+            }
+
+            #[bench]
+            fn bench_insert_bucket_map(bencher: &mut Bencher) {
+                do_bench_insert_bucket_map(bencher, $n, $m);
+            }
+        }
+    };
+}
+
+extern crate test;
+use rayon::prelude::*;
+use solana_bucket_map::bucket_map::{BucketMap, BucketMapConfig};
+use solana_sdk::pubkey::Pubkey;
+use std::collections::hash_map::HashMap;
+use std::sync::RwLock;
+use test::Bencher;
+
+type IndexValue = u64;
+
+DEFINE_NxM_BENCH!(dim_01x02, 1, 2);
+DEFINE_NxM_BENCH!(dim_02x04, 2, 4);
+DEFINE_NxM_BENCH!(dim_04x08, 4, 8);
+DEFINE_NxM_BENCH!(dim_08x16, 8, 16);
+DEFINE_NxM_BENCH!(dim_16x32, 16, 32);
+DEFINE_NxM_BENCH!(dim_32x64, 32, 64);
+
+/// Benchmark insert with Hashmap as baseline for N threads inserting M keys each
+fn do_bench_insert_baseline_hashmap(bencher: &mut Bencher, n: usize, m: usize) {
+    let index = RwLock::new(HashMap::new());
+    (0..n).into_iter().into_par_iter().for_each(|i| {
+        let key = Pubkey::new_unique();
+        index
+            .write()
+            .unwrap()
+            .insert(key, vec![(i, IndexValue::default())]);
+    });
+    bencher.iter(|| {
+        (0..n).into_iter().into_par_iter().for_each(|_| {
+            for j in 0..m {
+                let key = Pubkey::new_unique();
+                index
+                    .write()
+                    .unwrap()
+                    .insert(key, vec![(j, IndexValue::default())]);
+            }
+        })
+    });
+}
+
+/// Benchmark insert with BucketMap with N buckets for N threads inserting M keys each
+fn do_bench_insert_bucket_map(bencher: &mut Bencher, n: usize, m: usize) {
+    let index = BucketMap::new(BucketMapConfig::new(n));
+    (0..n).into_iter().into_par_iter().for_each(|i| {
+        let key = Pubkey::new_unique();
+        index.update(&key, |_| Some((vec![(i, IndexValue::default())], 0)));
+    });
+    bencher.iter(|| {
+        (0..n).into_iter().into_par_iter().for_each(|_| {
+            for j in 0..m {
+                let key = Pubkey::new_unique();
+                index.update(&key, |_| Some((vec![(j, IndexValue::default())], 0)));
+            }
+        })
+    });
+}