introduce bucket map (#19848)

* introduce bucket map

* rename BucketMap bits to num_buckets_pow2

* use u64::BITS

* Store the number of buckets in BucketMapConfig as a regular number

* remove redundant type aliases

* use Slot from sdk

* use Arc::clone() instead

* fixup erase drives

* rename num_buckets to max_buckets

* add doc to BucketMapConfig::new()

* add more documentation

* rename to DEFAULT_CAPACITY_POW2

* doc

* add more traits while we can

* rename capacity to capacity_pow2

* fix a naming for max_buckets_pow2

* remove unused/incorrect DataBucket::bytes

* rework benches a bit

* fixup bench docs

* rename create_bucket_capacity_pow2 to bucket_capacity_when_created_pow2

* rename BucketMapKeyValue to BucketItem

* rename to items_in_range

* remove values()

* remove addref and unref

* remove more addref and unref

* resurect addref and unref since tests use 'em for now

* rename to BucketStorage

* move stats in bucket_stats

* remove specializations (i don't think they are needed)

* move MaxSearch and RefCount into lib.rs

* move BucketItem to bucket_item.rs

* add doc

* keys no longer returns an option

* Revert "remove specializations (i don't think they are needed)"

This reverts commit b22f78e072.

Co-authored-by: Brooks Prumo <brooks@solana.com>

bucket_map/tests/bucket_map.rs

@@ -0,0 +1,46 @@
+use rayon::prelude::*;
+use solana_bucket_map::bucket_map::{BucketMap, BucketMapConfig};
+use solana_measure::measure::Measure;
+use solana_sdk::pubkey::Pubkey;
+use std::path::PathBuf;
+#[test]
+#[ignore]
+fn bucket_map_test_mt() {
+    let threads = 4096;
+    let items = 4096;
+    let tmpdir1 = std::env::temp_dir().join("bucket_map_test_mt");
+    let tmpdir2 = PathBuf::from("/mnt/data/0").join("bucket_map_test_mt");
+    let paths: Vec<PathBuf> = [tmpdir1, tmpdir2]
+        .iter()
+        .filter(|x| std::fs::create_dir_all(x).is_ok())
+        .cloned()
+        .collect();
+    assert!(!paths.is_empty());
+    let index = BucketMap::new(BucketMapConfig {
+        max_buckets: 1 << 12,
+        drives: Some(paths.clone()),
+        ..BucketMapConfig::default()
+    });
+    (0..threads).into_iter().into_par_iter().for_each(|_| {
+        let key = Pubkey::new_unique();
+        index.update(&key, |_| Some((vec![0u64], 0)));
+    });
+    let mut timer = Measure::start("bucket_map_test_mt");
+    (0..threads).into_iter().into_par_iter().for_each(|_| {
+        for _ in 0..items {
+            let key = Pubkey::new_unique();
+            let ix: u64 = index.bucket_ix(&key) as u64;
+            index.update(&key, |_| Some((vec![ix], 0)));
+            assert_eq!(index.read_value(&key), Some((vec![ix], 0)));
+        }
+    });
+    timer.stop();
+    println!("time: {}ns per item", timer.as_ns() / (threads * items));
+    let mut total = 0;
+    for tmpdir in paths.iter() {
+        let folder_size = fs_extra::dir::get_size(tmpdir).unwrap();
+        total += folder_size;
+        std::fs::remove_dir_all(tmpdir).unwrap();
+    }
+    println!("overhead: {}bytes per item", total / (threads * items));
+}