From fbf9dc47e9907f6cd688ebb49f85362c75c1254e Mon Sep 17 00:00:00 2001 From: "Jeff Washington (jwash)" <75863576+jeffwashington@users.noreply.github.com> Date: Fri, 5 Feb 2021 18:48:16 -0600 Subject: [PATCH] Threadpool2 (#15151) * rework thread pool for hash calculation * rename --- core/src/validator.rs | 2 +- ledger-tool/src/main.rs | 2 +- runtime/src/accounts_db.rs | 19 ++++++++++--------- runtime/src/snapshot_utils.rs | 6 +++--- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/core/src/validator.rs b/core/src/validator.rs index b8f7b477fd..11db515fe9 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -1050,7 +1050,7 @@ fn new_banks_from_ledger( None, &snapshot_config.snapshot_package_output_path, snapshot_config.archive_format, - &bank_forks.root_bank().get_thread_pool(), + Some(&bank_forks.root_bank().get_thread_pool()), ) .unwrap_or_else(|err| { error!("Unable to create snapshot: {}", err); diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 0c8f15075a..0090010f64 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1933,7 +1933,7 @@ fn main() { Some(snapshot_version), output_directory, ArchiveFormat::TarZstd, - &bank.get_thread_pool(), + None, ) .unwrap_or_else(|err| { eprintln!("Unable to create snapshot: {}", err); diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index e2839901d7..e9daed4ff8 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -3748,7 +3748,7 @@ impl AccountsDB { Self::calculate_accounts_hash_without_index( &combined_maps, simple_capitalization_enabled, - &self.thread_pool_clean, + Some(&self.thread_pool_clean), ) } else { self.calculate_accounts_hash(slot, ancestors, false, simple_capitalization_enabled) @@ -3852,13 +3852,18 @@ impl AccountsDB { pub fn calculate_accounts_hash_without_index( storages: &[SnapshotStorage], simple_capitalization_enabled: bool, - thread_pool: &ThreadPool, + thread_pool: Option<&ThreadPool>, ) -> (Hash, u64) { - thread_pool.install(|| { + let scan_and_hash = || { let result = Self::scan_snapshot_stores(storages, simple_capitalization_enabled); Self::rest_of_hash_calculation(result) - }) + }; + if let Some(thread_pool) = thread_pool { + thread_pool.install(scan_and_hash) + } else { + scan_and_hash() + } } pub fn verify_bank_hash_and_lamports( @@ -5184,11 +5189,7 @@ pub mod tests { solana_logger::setup(); let (storages, _size, _slot_expected) = sample_storage(); - let result = AccountsDB::calculate_accounts_hash_without_index( - &storages, - true, - &make_min_priority_thread_pool(), - ); + let result = AccountsDB::calculate_accounts_hash_without_index(&storages, true, None); let expected_hash = Hash::from_str("GKot5hBsd81kMupNCXHaqbhv3huEbxAFMLnpcX2hniwn").unwrap(); assert_eq!(result, (expected_hash, 0)); } diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index db6fe93287..57dfc67e13 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -927,7 +927,7 @@ pub fn bank_to_snapshot_archive, Q: AsRef>( snapshot_version: Option, snapshot_package_output_path: Q, archive_format: ArchiveFormat, - thread_pool: &ThreadPool, + thread_pool: Option<&ThreadPool>, ) -> Result { let snapshot_version = snapshot_version.unwrap_or_default(); @@ -954,7 +954,7 @@ pub fn bank_to_snapshot_archive, Q: AsRef>( None, )?; - let package = process_accounts_package_pre(package, Some(&thread_pool)); + let package = process_accounts_package_pre(package, thread_pool); archive_snapshot_package(&package)?; Ok(package.tar_output_file) @@ -971,7 +971,7 @@ pub fn process_accounts_package_pre( let (hash, lamports) = AccountsDB::calculate_accounts_hash_without_index( &accounts_package.storages, accounts_package.simple_capitalization_testing, - &thread_pool.unwrap(), + thread_pool, ); assert_eq!(accounts_package.expected_capitalization, lamports);