Threadpool2 (#15151)
* rework thread pool for hash calculation * rename
This commit is contained in:
committed by
GitHub
parent
3a5c142a9b
commit
fbf9dc47e9
@ -1050,7 +1050,7 @@ fn new_banks_from_ledger(
|
|||||||
None,
|
None,
|
||||||
&snapshot_config.snapshot_package_output_path,
|
&snapshot_config.snapshot_package_output_path,
|
||||||
snapshot_config.archive_format,
|
snapshot_config.archive_format,
|
||||||
&bank_forks.root_bank().get_thread_pool(),
|
Some(&bank_forks.root_bank().get_thread_pool()),
|
||||||
)
|
)
|
||||||
.unwrap_or_else(|err| {
|
.unwrap_or_else(|err| {
|
||||||
error!("Unable to create snapshot: {}", err);
|
error!("Unable to create snapshot: {}", err);
|
||||||
|
@ -1933,7 +1933,7 @@ fn main() {
|
|||||||
Some(snapshot_version),
|
Some(snapshot_version),
|
||||||
output_directory,
|
output_directory,
|
||||||
ArchiveFormat::TarZstd,
|
ArchiveFormat::TarZstd,
|
||||||
&bank.get_thread_pool(),
|
None,
|
||||||
)
|
)
|
||||||
.unwrap_or_else(|err| {
|
.unwrap_or_else(|err| {
|
||||||
eprintln!("Unable to create snapshot: {}", err);
|
eprintln!("Unable to create snapshot: {}", err);
|
||||||
|
@ -3748,7 +3748,7 @@ impl AccountsDB {
|
|||||||
Self::calculate_accounts_hash_without_index(
|
Self::calculate_accounts_hash_without_index(
|
||||||
&combined_maps,
|
&combined_maps,
|
||||||
simple_capitalization_enabled,
|
simple_capitalization_enabled,
|
||||||
&self.thread_pool_clean,
|
Some(&self.thread_pool_clean),
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
self.calculate_accounts_hash(slot, ancestors, false, simple_capitalization_enabled)
|
self.calculate_accounts_hash(slot, ancestors, false, simple_capitalization_enabled)
|
||||||
@ -3852,13 +3852,18 @@ impl AccountsDB {
|
|||||||
pub fn calculate_accounts_hash_without_index(
|
pub fn calculate_accounts_hash_without_index(
|
||||||
storages: &[SnapshotStorage],
|
storages: &[SnapshotStorage],
|
||||||
simple_capitalization_enabled: bool,
|
simple_capitalization_enabled: bool,
|
||||||
thread_pool: &ThreadPool,
|
thread_pool: Option<&ThreadPool>,
|
||||||
) -> (Hash, u64) {
|
) -> (Hash, u64) {
|
||||||
thread_pool.install(|| {
|
let scan_and_hash = || {
|
||||||
let result = Self::scan_snapshot_stores(storages, simple_capitalization_enabled);
|
let result = Self::scan_snapshot_stores(storages, simple_capitalization_enabled);
|
||||||
|
|
||||||
Self::rest_of_hash_calculation(result)
|
Self::rest_of_hash_calculation(result)
|
||||||
})
|
};
|
||||||
|
if let Some(thread_pool) = thread_pool {
|
||||||
|
thread_pool.install(scan_and_hash)
|
||||||
|
} else {
|
||||||
|
scan_and_hash()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn verify_bank_hash_and_lamports(
|
pub fn verify_bank_hash_and_lamports(
|
||||||
@ -5184,11 +5189,7 @@ pub mod tests {
|
|||||||
solana_logger::setup();
|
solana_logger::setup();
|
||||||
|
|
||||||
let (storages, _size, _slot_expected) = sample_storage();
|
let (storages, _size, _slot_expected) = sample_storage();
|
||||||
let result = AccountsDB::calculate_accounts_hash_without_index(
|
let result = AccountsDB::calculate_accounts_hash_without_index(&storages, true, None);
|
||||||
&storages,
|
|
||||||
true,
|
|
||||||
&make_min_priority_thread_pool(),
|
|
||||||
);
|
|
||||||
let expected_hash = Hash::from_str("GKot5hBsd81kMupNCXHaqbhv3huEbxAFMLnpcX2hniwn").unwrap();
|
let expected_hash = Hash::from_str("GKot5hBsd81kMupNCXHaqbhv3huEbxAFMLnpcX2hniwn").unwrap();
|
||||||
assert_eq!(result, (expected_hash, 0));
|
assert_eq!(result, (expected_hash, 0));
|
||||||
}
|
}
|
||||||
|
@ -927,7 +927,7 @@ pub fn bank_to_snapshot_archive<P: AsRef<Path>, Q: AsRef<Path>>(
|
|||||||
snapshot_version: Option<SnapshotVersion>,
|
snapshot_version: Option<SnapshotVersion>,
|
||||||
snapshot_package_output_path: Q,
|
snapshot_package_output_path: Q,
|
||||||
archive_format: ArchiveFormat,
|
archive_format: ArchiveFormat,
|
||||||
thread_pool: &ThreadPool,
|
thread_pool: Option<&ThreadPool>,
|
||||||
) -> Result<PathBuf> {
|
) -> Result<PathBuf> {
|
||||||
let snapshot_version = snapshot_version.unwrap_or_default();
|
let snapshot_version = snapshot_version.unwrap_or_default();
|
||||||
|
|
||||||
@ -954,7 +954,7 @@ pub fn bank_to_snapshot_archive<P: AsRef<Path>, Q: AsRef<Path>>(
|
|||||||
None,
|
None,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
let package = process_accounts_package_pre(package, Some(&thread_pool));
|
let package = process_accounts_package_pre(package, thread_pool);
|
||||||
|
|
||||||
archive_snapshot_package(&package)?;
|
archive_snapshot_package(&package)?;
|
||||||
Ok(package.tar_output_file)
|
Ok(package.tar_output_file)
|
||||||
@ -971,7 +971,7 @@ pub fn process_accounts_package_pre(
|
|||||||
let (hash, lamports) = AccountsDB::calculate_accounts_hash_without_index(
|
let (hash, lamports) = AccountsDB::calculate_accounts_hash_without_index(
|
||||||
&accounts_package.storages,
|
&accounts_package.storages,
|
||||||
accounts_package.simple_capitalization_testing,
|
accounts_package.simple_capitalization_testing,
|
||||||
&thread_pool.unwrap(),
|
thread_pool,
|
||||||
);
|
);
|
||||||
|
|
||||||
assert_eq!(accounts_package.expected_capitalization, lamports);
|
assert_eq!(accounts_package.expected_capitalization, lamports);
|
||||||
|
Reference in New Issue
Block a user