Remove redundant threadpools in sigverify (#7888)

* Limit the number of thread pools sigverify creates

* Name local threadpools

Cargo.lock

@@ -3979,6 +3979,7 @@ dependencies = [
  "ed25519-dalek 1.0.0-pre.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)",

ledger/Cargo.toml

@@ -46,6 +46,7 @@ sys-info = "0.5.8"
 tar = "0.4.26"
 thiserror = "1.0"
 tempfile = "3.1.0"
+lazy_static = "1.4.0"
 
 [dependencies.rocksdb]
 # Avoid the vendored bzip2 within rocksdb-sys that can cause linker conflicts

ledger/src/blockstore.rs

@@ -58,6 +58,7 @@ pub const BLOCKSTORE_DIRECTORY: &str = "rocksdb";
 
 thread_local!(static PAR_THREAD_POOL: RefCell<ThreadPool> = RefCell::new(rayon::ThreadPoolBuilder::new()
                     .num_threads(get_thread_count())
+                    .thread_name(|ix| format!("blockstore_{}", ix))
                     .build()
                     .unwrap()));
 

ledger/src/blockstore_processor.rs

@@ -39,6 +39,7 @@ use thiserror::Error;
 
 thread_local!(static PAR_THREAD_POOL: RefCell<ThreadPool> = RefCell::new(rayon::ThreadPoolBuilder::new()
                     .num_threads(get_thread_count())
+                    .thread_name(|ix| format!("blockstore_processor_{}", ix))
                     .build()
                     .unwrap())
 );

ledger/src/entry.rs

@@ -26,6 +26,7 @@ use std::{cmp, thread};
 
 thread_local!(static PAR_THREAD_POOL: RefCell<ThreadPool> = RefCell::new(rayon::ThreadPoolBuilder::new()
                     .num_threads(get_thread_count())
+                    .thread_name(|ix| format!("entry_{}", ix))
                     .build()
                     .unwrap()));
 

ledger/src/lib.rs

@@ -25,3 +25,6 @@ extern crate solana_metrics;
 
 #[macro_use]
 extern crate log;
+
+#[macro_use]
+extern crate lazy_static;

ledger/src/shred.rs

@@ -46,6 +46,7 @@ pub const OFFSET_OF_SHRED_INDEX: usize = OFFSET_OF_SHRED_SLOT + SIZE_OF_SHRED_SL
 
 thread_local!(static PAR_THREAD_POOL: RefCell<ThreadPool> = RefCell::new(rayon::ThreadPoolBuilder::new()
                     .num_threads(get_thread_count())
+                    .thread_name(|ix| format!("shredder_{}", ix))
                     .build()
                     .unwrap()));
 

ledger/src/sigverify_shreds.rs

@@ -20,15 +20,17 @@ use solana_sdk::pubkey::Pubkey;
 use solana_sdk::signature::Signature;
 use solana_sdk::signature::{Keypair, KeypairUtil};
 use std::sync::Arc;
-use std::{cell::RefCell, collections::HashMap, mem::size_of};
+use std::{collections::HashMap, mem::size_of};
 
 pub const SIGN_SHRED_GPU_MIN: usize = 256;
 
-thread_local!(static PAR_THREAD_POOL: RefCell<ThreadPool> = RefCell::new(rayon::ThreadPoolBuilder::new()
+lazy_static! {
+    pub static ref SIGVERIFY_THREAD_POOL: ThreadPool = rayon::ThreadPoolBuilder::new()
         .num_threads(get_thread_count())
         .thread_name(|ix| format!("sigverify_shreds_{}", ix))
         .build()
-                    .unwrap()));
+        .unwrap();
+}
 
 /// Assuming layout is
 /// signature: Signature
@@ -70,8 +72,7 @@ fn verify_shreds_cpu(batches: &[Packets], slot_leaders: &HashMap<u64, [u8; 32]>)
     use rayon::prelude::*;
     let count = batch_size(batches);
     debug!("CPU SHRED ECDSA for {}", count);
-    let rv = PAR_THREAD_POOL.with(|thread_pool| {
+    let rv = SIGVERIFY_THREAD_POOL.install(|| {
-        thread_pool.borrow().install(|| {
         batches
             .into_par_iter()
             .map(|p| {
@@ -81,7 +82,6 @@ fn verify_shreds_cpu(batches: &[Packets], slot_leaders: &HashMap<u64, [u8; 32]>)
                     .collect()
             })
             .collect()
-        })
     });
     inc_new_counter_debug!("ed25519_shred_verify_cpu", count);
     rv
@@ -97,8 +97,7 @@ fn slot_key_data_for_gpu<
 ) -> (PinnedVec<u8>, TxOffset, usize) {
     //TODO: mark Pubkey::default shreds as failed after the GPU returns
     assert_eq!(slot_keys.get(&std::u64::MAX), Some(&T::default()));
-    let slots: Vec<Vec<u64>> = PAR_THREAD_POOL.with(|thread_pool| {
+    let slots: Vec<Vec<u64>> = SIGVERIFY_THREAD_POOL.install(|| {
-        thread_pool.borrow().install(|| {
         batches
             .into_par_iter()
             .map(|p| {
@@ -120,7 +119,6 @@ fn slot_key_data_for_gpu<
                     .collect()
             })
             .collect()
-        })
     });
     let mut keys_to_slots: HashMap<T, Vec<u64>> = HashMap::new();
     for batch in slots.iter() {
@@ -312,14 +310,12 @@ pub fn sign_shreds_cpu(keypair: &Keypair, batches: &mut [Packets]) {
     use rayon::prelude::*;
     let count = batch_size(batches);
     debug!("CPU SHRED ECDSA for {}", count);
-    PAR_THREAD_POOL.with(|thread_pool| {
+    SIGVERIFY_THREAD_POOL.install(|| {
-        thread_pool.borrow().install(|| {
         batches.par_iter_mut().for_each(|p| {
             p.packets[..]
                 .par_iter_mut()
                 .for_each(|mut p| sign_shred_cpu(keypair, &mut p));
         });
-        })
     });
     inc_new_counter_debug!("ed25519_shred_verify_cpu", count);
 }
@@ -425,8 +421,7 @@ pub fn sign_shreds_gpu(
         }
         sizes[i] += sizes[i - 1];
     }
-    PAR_THREAD_POOL.with(|thread_pool| {
+    SIGVERIFY_THREAD_POOL.install(|| {
-        thread_pool.borrow().install(|| {
         batches
             .par_iter_mut()
             .enumerate()
@@ -444,7 +439,6 @@ pub fn sign_shreds_gpu(
                     });
             });
     });
-    });
     inc_new_counter_debug!("ed25519_shred_sign_gpu", count);
 }
 

perf/src/sigverify.rs

@@ -18,14 +18,15 @@ use solana_sdk::short_vec::decode_len;
 use solana_sdk::signature::Signature;
 #[cfg(test)]
 use solana_sdk::transaction::Transaction;
-use std::cell::RefCell;
 use std::mem::size_of;
 
-thread_local!(static PAR_THREAD_POOL: RefCell<ThreadPool> = RefCell::new(rayon::ThreadPoolBuilder::new()
+lazy_static! {
+    static ref PAR_THREAD_POOL: ThreadPool = rayon::ThreadPoolBuilder::new()
         .num_threads(get_thread_count())
         .thread_name(|ix| format!("sigverify_{}", ix))
         .build()
-                    .unwrap()));
+        .unwrap();
+}
 
 pub type TxOffset = PinnedVec<u32>;
 
@@ -247,13 +248,11 @@ pub fn ed25519_verify_cpu(batches: &[Packets]) -> Vec<Vec<u8>> {
     use rayon::prelude::*;
     let count = batch_size(batches);
     debug!("CPU ECDSA for {}", batch_size(batches));
-    let rv = PAR_THREAD_POOL.with(|thread_pool| {
+    let rv = PAR_THREAD_POOL.install(|| {
-        thread_pool.borrow().install(|| {
         batches
             .into_par_iter()
             .map(|p| p.packets.par_iter().map(verify_packet).collect())
             .collect()
-        })
     });
     inc_new_counter_debug!("ed25519_verify_cpu", count);
     rv