improves threads' utilization in processing gossip packets (#12962) (#13023)

ClusterInfo::process_packets handles incoming packets in a thread_pool:
https://github.com/solana-labs/solana/blob/87311cce7/core/src/cluster_info.rs#L2118-L2134

However, profiling runtime shows that threads are not well utilized and
a lot of the processing is done sequentially.

This commit redistributes the work done in parallel. Testing on a gce
cluster shows 20%+ improvement in processing gossip packets with much
smaller variations.

(cherry picked from commit 75d62ca095)

Co-authored-by: behzad nouri <behzadnouri@gmail.com>
This commit is contained in:
mergify[bot]
2020-10-20 19:59:35 +00:00
committed by GitHub
parent 9da2ac7a44
commit d05bfa08c7
3 changed files with 174 additions and 114 deletions

View File

@@ -165,6 +165,21 @@ impl<'a, T: Clone + Send + Sync + Default + Sized> IntoParallelIterator for &'a
}
}
impl<T: Clone + Default + Send + Sized> IntoParallelIterator for PinnedVec<T> {
type Item = T;
type Iter = rayon::vec::IntoIter<T>;
fn into_par_iter(mut self) -> Self::Iter {
if self.pinned {
unpin(self.x.as_mut_ptr());
self.pinned = false;
}
self.pinnable = false;
self.recycler = None;
std::mem::take(&mut self.x).into_par_iter()
}
}
impl<T: Clone + Default + Sized> PinnedVec<T> {
pub fn reserve_and_pin(&mut self, size: usize) {
if self.x.capacity() < size {