builds crds filters in parallel (#12360)

Based on run-time profiles, the majority time of new_pull_requests is
spent building bloom filters, in hashing and bit-vec ops.

This commit builds crds filters in parallel using rayon constructs. The
added benchmark shows ~5x speedup (4-core machine, 8 threads).
This commit is contained in:
behzad nouri
2020-09-29 23:06:02 +00:00
committed by GitHub
parent 96a7d4dbd8
commit 537bbde22e
7 changed files with 270 additions and 99 deletions

View File

@@ -3,19 +3,19 @@
extern crate test;
use rand::{thread_rng, Rng};
use solana_core::crds_gossip_pull::CrdsFilter;
use solana_sdk::hash::{Hash, HASH_BYTES};
use rayon::ThreadPoolBuilder;
use solana_core::cluster_info::MAX_BLOOM_SIZE;
use solana_core::crds::Crds;
use solana_core::crds_gossip_pull::{CrdsFilter, CrdsGossipPull};
use solana_core::crds_value::CrdsValue;
use solana_sdk::hash::Hash;
use test::Bencher;
#[bench]
fn bench_hash_as_u64(bencher: &mut Bencher) {
let mut rng = thread_rng();
let hashes: Vec<_> = (0..1000)
.map(|_| {
let mut buf = [0u8; HASH_BYTES];
rng.fill(&mut buf);
Hash::new(&buf)
})
let hashes: Vec<_> = std::iter::repeat_with(|| Hash::new_rand(&mut rng))
.take(1000)
.collect();
bencher.iter(|| {
hashes
@@ -24,3 +24,30 @@ fn bench_hash_as_u64(bencher: &mut Bencher) {
.collect::<Vec<_>>()
});
}
#[bench]
fn bench_build_crds_filters(bencher: &mut Bencher) {
let thread_pool = ThreadPoolBuilder::new().build().unwrap();
let mut rng = thread_rng();
let mut crds_gossip_pull = CrdsGossipPull::default();
let mut crds = Crds::default();
for _ in 0..50_000 {
crds_gossip_pull
.purged_values
.push_back((Hash::new_rand(&mut rng), rng.gen()));
}
let mut num_inserts = 0;
for _ in 0..90_000 {
if crds
.insert(CrdsValue::new_rand(&mut rng), rng.gen())
.is_ok()
{
num_inserts += 1;
}
}
assert_eq!(num_inserts, 90_000);
bencher.iter(|| {
let filters = crds_gossip_pull.build_crds_filters(&thread_pool, &crds, MAX_BLOOM_SIZE);
assert_eq!(filters.len(), 128);
});
}