Threadpool2 (#15151)

* rework thread pool for hash calculation

* rename
This commit is contained in:
Jeff Washington (jwash)
2021-02-05 18:48:16 -06:00
committed by GitHub
parent 3a5c142a9b
commit fbf9dc47e9
4 changed files with 15 additions and 14 deletions

View File

@ -1050,7 +1050,7 @@ fn new_banks_from_ledger(
None, None,
&snapshot_config.snapshot_package_output_path, &snapshot_config.snapshot_package_output_path,
snapshot_config.archive_format, snapshot_config.archive_format,
&bank_forks.root_bank().get_thread_pool(), Some(&bank_forks.root_bank().get_thread_pool()),
) )
.unwrap_or_else(|err| { .unwrap_or_else(|err| {
error!("Unable to create snapshot: {}", err); error!("Unable to create snapshot: {}", err);

View File

@ -1933,7 +1933,7 @@ fn main() {
Some(snapshot_version), Some(snapshot_version),
output_directory, output_directory,
ArchiveFormat::TarZstd, ArchiveFormat::TarZstd,
&bank.get_thread_pool(), None,
) )
.unwrap_or_else(|err| { .unwrap_or_else(|err| {
eprintln!("Unable to create snapshot: {}", err); eprintln!("Unable to create snapshot: {}", err);

View File

@ -3748,7 +3748,7 @@ impl AccountsDB {
Self::calculate_accounts_hash_without_index( Self::calculate_accounts_hash_without_index(
&combined_maps, &combined_maps,
simple_capitalization_enabled, simple_capitalization_enabled,
&self.thread_pool_clean, Some(&self.thread_pool_clean),
) )
} else { } else {
self.calculate_accounts_hash(slot, ancestors, false, simple_capitalization_enabled) self.calculate_accounts_hash(slot, ancestors, false, simple_capitalization_enabled)
@ -3852,13 +3852,18 @@ impl AccountsDB {
pub fn calculate_accounts_hash_without_index( pub fn calculate_accounts_hash_without_index(
storages: &[SnapshotStorage], storages: &[SnapshotStorage],
simple_capitalization_enabled: bool, simple_capitalization_enabled: bool,
thread_pool: &ThreadPool, thread_pool: Option<&ThreadPool>,
) -> (Hash, u64) { ) -> (Hash, u64) {
thread_pool.install(|| { let scan_and_hash = || {
let result = Self::scan_snapshot_stores(storages, simple_capitalization_enabled); let result = Self::scan_snapshot_stores(storages, simple_capitalization_enabled);
Self::rest_of_hash_calculation(result) Self::rest_of_hash_calculation(result)
}) };
if let Some(thread_pool) = thread_pool {
thread_pool.install(scan_and_hash)
} else {
scan_and_hash()
}
} }
pub fn verify_bank_hash_and_lamports( pub fn verify_bank_hash_and_lamports(
@ -5184,11 +5189,7 @@ pub mod tests {
solana_logger::setup(); solana_logger::setup();
let (storages, _size, _slot_expected) = sample_storage(); let (storages, _size, _slot_expected) = sample_storage();
let result = AccountsDB::calculate_accounts_hash_without_index( let result = AccountsDB::calculate_accounts_hash_without_index(&storages, true, None);
&storages,
true,
&make_min_priority_thread_pool(),
);
let expected_hash = Hash::from_str("GKot5hBsd81kMupNCXHaqbhv3huEbxAFMLnpcX2hniwn").unwrap(); let expected_hash = Hash::from_str("GKot5hBsd81kMupNCXHaqbhv3huEbxAFMLnpcX2hniwn").unwrap();
assert_eq!(result, (expected_hash, 0)); assert_eq!(result, (expected_hash, 0));
} }

View File

@ -927,7 +927,7 @@ pub fn bank_to_snapshot_archive<P: AsRef<Path>, Q: AsRef<Path>>(
snapshot_version: Option<SnapshotVersion>, snapshot_version: Option<SnapshotVersion>,
snapshot_package_output_path: Q, snapshot_package_output_path: Q,
archive_format: ArchiveFormat, archive_format: ArchiveFormat,
thread_pool: &ThreadPool, thread_pool: Option<&ThreadPool>,
) -> Result<PathBuf> { ) -> Result<PathBuf> {
let snapshot_version = snapshot_version.unwrap_or_default(); let snapshot_version = snapshot_version.unwrap_or_default();
@ -954,7 +954,7 @@ pub fn bank_to_snapshot_archive<P: AsRef<Path>, Q: AsRef<Path>>(
None, None,
)?; )?;
let package = process_accounts_package_pre(package, Some(&thread_pool)); let package = process_accounts_package_pre(package, thread_pool);
archive_snapshot_package(&package)?; archive_snapshot_package(&package)?;
Ok(package.tar_output_file) Ok(package.tar_output_file)
@ -971,7 +971,7 @@ pub fn process_accounts_package_pre(
let (hash, lamports) = AccountsDB::calculate_accounts_hash_without_index( let (hash, lamports) = AccountsDB::calculate_accounts_hash_without_index(
&accounts_package.storages, &accounts_package.storages,
accounts_package.simple_capitalization_testing, accounts_package.simple_capitalization_testing,
&thread_pool.unwrap(), thread_pool,
); );
assert_eq!(accounts_package.expected_capitalization, lamports); assert_eq!(accounts_package.expected_capitalization, lamports);