4e9e05f311
* shards crds values based on their hash prefix (#12187)
filter_crds_values checks every crds filter against every hash value:
https://github.com/solana-labs/solana/blob/ee646aa7/core/src/crds_gossip_pull.rs#L432
which can be inefficient if the filter's bit-mask only matches small
portion of the entire crds table.
This commit shards crds values into separate tables based on shard_bits
first bits of their hash prefix. Given a (mask, mask_bits) filter,
filtering crds can be done by inspecting only relevant shards.
If CrdsFilter.mask_bits <= shard_bits, then precisely only the crds
values which match (mask, mask_bits) bit pattern are traversed.
If CrdsFilter.mask_bits > shard_bits, then approximately only
1/2^shard_bits of crds values are inspected.
Benchmarking on a gce cluster of 20 nodes, I see ~10% improvement in
generate_pull_responses metric, but with larger clusters, crds table and
2^mask_bits are both larger, so the impact should be more significant.
(cherry picked from commit 9b866d79fb)
* bumps indexmap to 1.6.0
Co-authored-by: behzad nouri <behzadnouri@gmail.com>
70 lines
1.9 KiB
Rust
70 lines
1.9 KiB
Rust
#![feature(test)]
|
|
|
|
extern crate test;
|
|
|
|
use rand::{thread_rng, Rng};
|
|
use solana_core::contact_info::ContactInfo;
|
|
use solana_core::crds::VersionedCrdsValue;
|
|
use solana_core::crds_shards::CrdsShards;
|
|
use solana_core::crds_value::{CrdsData, CrdsValue};
|
|
use solana_sdk::pubkey::Pubkey;
|
|
use solana_sdk::timing::timestamp;
|
|
use test::Bencher;
|
|
|
|
const CRDS_SHARDS_BITS: u32 = 8;
|
|
|
|
fn new_test_crds_value() -> VersionedCrdsValue {
|
|
let data = CrdsData::ContactInfo(ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp()));
|
|
VersionedCrdsValue::new(timestamp(), CrdsValue::new_unsigned(data))
|
|
}
|
|
|
|
fn bench_crds_shards_find(bencher: &mut Bencher, num_values: usize, mask_bits: u32) {
|
|
let values: Vec<VersionedCrdsValue> = std::iter::repeat_with(new_test_crds_value)
|
|
.take(num_values)
|
|
.collect();
|
|
let mut shards = CrdsShards::new(CRDS_SHARDS_BITS);
|
|
for (index, value) in values.iter().enumerate() {
|
|
assert!(shards.insert(index, value));
|
|
}
|
|
let mut rng = thread_rng();
|
|
bencher.iter(|| {
|
|
let mask = rng.gen();
|
|
let _hits = shards.find(mask, mask_bits).count();
|
|
});
|
|
}
|
|
|
|
#[bench]
|
|
fn bench_crds_shards_find_0(bencher: &mut Bencher) {
|
|
bench_crds_shards_find(bencher, 100_000, 0);
|
|
}
|
|
|
|
#[bench]
|
|
fn bench_crds_shards_find_1(bencher: &mut Bencher) {
|
|
bench_crds_shards_find(bencher, 100_000, 1);
|
|
}
|
|
|
|
#[bench]
|
|
fn bench_crds_shards_find_3(bencher: &mut Bencher) {
|
|
bench_crds_shards_find(bencher, 100_000, 3);
|
|
}
|
|
|
|
#[bench]
|
|
fn bench_crds_shards_find_5(bencher: &mut Bencher) {
|
|
bench_crds_shards_find(bencher, 100_000, 5);
|
|
}
|
|
|
|
#[bench]
|
|
fn bench_crds_shards_find_7(bencher: &mut Bencher) {
|
|
bench_crds_shards_find(bencher, 100_000, 7);
|
|
}
|
|
|
|
#[bench]
|
|
fn bench_crds_shards_find_8(bencher: &mut Bencher) {
|
|
bench_crds_shards_find(bencher, 100_000, 8);
|
|
}
|
|
|
|
#[bench]
|
|
fn bench_crds_shards_find_9(bencher: &mut Bencher) {
|
|
bench_crds_shards_find(bencher, 100_000, 9);
|
|
}
|