gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add AccountsHashConfig to manage parameters (#23850)

Browse Source
This commit is contained in:
Jeff Washington (jwash)
2022-03-23 13:44:23 -05:00
committed by GitHub
parent db49b826f0
commit 9e61fe7583
3 changed files with 84 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
core/src/accounts_hash_verifier.rs
View File

@ -10,8 +10,8 @@ use {
solana_gossip::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES}, solana_gossip::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES},
solana_measure::measure::Measure, solana_measure::measure::Measure,
solana_runtime::{ solana_runtime::{
accounts_db::{self}, accounts_db,
accounts_hash::HashStats, accounts_hash::{CalcAccountsHashConfig, HashStats},
snapshot_config::SnapshotConfig, snapshot_config::SnapshotConfig,
snapshot_package::{ snapshot_package::{
AccountsPackage, AccountsPackageReceiver, PendingSnapshotPackage, SnapshotPackage, AccountsPackage, AccountsPackageReceiver, PendingSnapshotPackage, SnapshotPackage,
@ -132,16 +132,16 @@ impl AccountsHashVerifier {
let (hash, lamports) = accounts_package let (hash, lamports) = accounts_package
.accounts .accounts
.accounts_db .accounts_db
.calculate_accounts_hash_without_index( .calculate_accounts_hash_without_index(&mut CalcAccountsHashConfig {
ledger_path, accounts_hash_cache_path: ledger_path,
&sorted_storages, storages: &sorted_storages,
thread_pool, thread_pool,
HashStats::default(), stats: HashStats::default(),
false, check_hash: false,
None, accounts_cache_and_ancestors: None,
None, // this will fail with filler accounts filler_account_suffix: None, // this will fail with filler accounts
None, // this code path is only for testing, so use default # passes here num_hash_scan_passes: None, // this code path is only for testing, so use default # passes here
) })
.unwrap(); .unwrap();
assert_eq!(accounts_package.expected_capitalization, lamports); assert_eq!(accounts_package.expected_capitalization, lamports);

96
runtime/src/accounts_db.rs
View File

@ -23,7 +23,10 @@ use {
account_info::{AccountInfo, Offset, StorageLocation, StoredSize}, account_info::{AccountInfo, Offset, StorageLocation, StoredSize},
accounts_background_service::{DroppedSlotsSender, SendDroppedBankCallback}, accounts_background_service::{DroppedSlotsSender, SendDroppedBankCallback},
accounts_cache::{AccountsCache, CachedAccount, SlotCache}, accounts_cache::{AccountsCache, CachedAccount, SlotCache},
accounts_hash::{AccountsHash, CalculateHashIntermediate, HashStats, PreviousPass}, accounts_hash::{
AccountsHash, CalcAccountsHashConfig, CalculateHashIntermediate, HashStats,
PreviousPass,
},
accounts_index::{ accounts_index::{
AccountIndexGetResult, AccountSecondaryIndexes, AccountsIndex, AccountsIndexConfig, AccountIndexGetResult, AccountSecondaryIndexes, AccountsIndex, AccountsIndexConfig,
AccountsIndexRootsStats, IndexKey, IndexValue, IsCached, RefCount, ScanConfig, AccountsIndexRootsStats, IndexKey, IndexValue, IsCached, RefCount, ScanConfig,
@ -978,7 +981,7 @@ struct RemoveUnrootedSlotsSynchronization {
signal: Condvar, signal: Condvar,
} }
type AccountInfoAccountsIndex = AccountsIndex<AccountInfo>; pub type AccountInfoAccountsIndex = AccountsIndex<AccountInfo>;
// This structure handles the load/store of the accounts // This structure handles the load/store of the accounts
#[derive(Debug)] #[derive(Debug)]
@ -5524,20 +5527,20 @@ impl AccountsDb {
} else { } else {
Some(&self.thread_pool_clean) Some(&self.thread_pool_clean)
}; };
self.calculate_accounts_hash_without_index( self.calculate_accounts_hash_without_index(&mut CalcAccountsHashConfig {
&self.accounts_hash_cache_path, accounts_hash_cache_path: &self.accounts_hash_cache_path,
&storages, storages: &storages,
thread_pool, thread_pool,
timings, stats: timings,
check_hash, check_hash,
accounts_cache_and_ancestors, accounts_cache_and_ancestors,
if self.filler_account_count > 0 { filler_account_suffix: if self.filler_account_count > 0 {
self.filler_account_suffix.as_ref() self.filler_account_suffix.as_ref()
} else { } else {
None None
}, },
self.num_hash_scan_passes, num_hash_scan_passes: self.num_hash_scan_passes,
) })
} else { } else {
self.calculate_accounts_hash(slot, ancestors, check_hash) self.calculate_accounts_hash(slot, ancestors, check_hash)
} }
@ -5729,25 +5732,16 @@ impl AccountsDb {
// intended to be faster than calculate_accounts_hash // intended to be faster than calculate_accounts_hash
pub fn calculate_accounts_hash_without_index( pub fn calculate_accounts_hash_without_index(
&self, &self,
accounts_hash_cache_path: &Path, config: &mut CalcAccountsHashConfig<'_>,
storages: &SortedStorages,
thread_pool: Option<&ThreadPool>,
mut stats: HashStats,
check_hash: bool,
accounts_cache_and_ancestors: Option<(
&AccountsCache,
&Ancestors,
&AccountInfoAccountsIndex,
)>,
filler_account_suffix: Option<&Pubkey>,
num_hash_scan_passes: Option<usize>,
) -> Result<(Hash, u64), BankHashVerificationError> { ) -> Result<(Hash, u64), BankHashVerificationError> {
let (num_hash_scan_passes, bins_per_pass) = Self::bins_per_pass(num_hash_scan_passes); let (num_hash_scan_passes, bins_per_pass) =
Self::bins_per_pass(config.num_hash_scan_passes);
let thread_pool = config.thread_pool;
let mut scan_and_hash = move || { let mut scan_and_hash = move || {
let mut previous_pass = PreviousPass::default(); let mut previous_pass = PreviousPass::default();
let mut final_result = (Hash::default(), 0); let mut final_result = (Hash::default(), 0);
let cache_hash_data = CacheHashData::new(&accounts_hash_cache_path); let cache_hash_data = CacheHashData::new(&config.accounts_hash_cache_path);
for pass in 0..num_hash_scan_passes { for pass in 0..num_hash_scan_passes {
let bounds = Range { let bounds = Range {
@ -5757,21 +5751,21 @@ impl AccountsDb {
let result = Self::scan_snapshot_stores_with_cache( let result = Self::scan_snapshot_stores_with_cache(
&cache_hash_data, &cache_hash_data,
storages, config.storages,
&mut stats, &mut config.stats,
PUBKEY_BINS_FOR_CALCULATING_HASHES, PUBKEY_BINS_FOR_CALCULATING_HASHES,
&bounds, &bounds,
check_hash, config.check_hash,
accounts_cache_and_ancestors, config.accounts_cache_and_ancestors,
filler_account_suffix, config.filler_account_suffix,
)?; )?;
let hash = AccountsHash { let hash = AccountsHash {
filler_account_suffix: filler_account_suffix.cloned(), filler_account_suffix: config.filler_account_suffix.cloned(),
}; };
let (hash, lamports, for_next_pass) = hash.rest_of_hash_calculation( let (hash, lamports, for_next_pass) = hash.rest_of_hash_calculation(
result, result,
&mut stats, &mut config.stats,
pass == num_hash_scan_passes - 1, pass == num_hash_scan_passes - 1,
previous_pass, previous_pass,
bins_per_pass, bins_per_pass,
@ -5782,7 +5776,7 @@ impl AccountsDb {
info!( info!(
"calculate_accounts_hash_without_index: slot (exclusive): {} {:?}", "calculate_accounts_hash_without_index: slot (exclusive): {} {:?}",
storages.range().end, config.storages.range().end,
final_result final_result
); );
Ok(final_result) Ok(final_result)
@ -7914,16 +7908,16 @@ pub mod tests {
let (storages, _size, _slot_expected) = sample_storage(); let (storages, _size, _slot_expected) = sample_storage();
let db = AccountsDb::new(Vec::new(), &ClusterType::Development); let db = AccountsDb::new(Vec::new(), &ClusterType::Development);
let result = db let result = db
.calculate_accounts_hash_without_index( .calculate_accounts_hash_without_index(&mut CalcAccountsHashConfig {
TempDir::new().unwrap().path(), accounts_hash_cache_path: TempDir::new().unwrap().path(),
&get_storage_refs(&storages), storages: &get_storage_refs(&storages),
None, thread_pool: None,
HashStats::default(), stats: HashStats::default(),
false, check_hash: false,
None, accounts_cache_and_ancestors: None,
None, filler_account_suffix: None,
None, num_hash_scan_passes: None,
) })
.unwrap(); .unwrap();
let expected_hash = Hash::from_str("GKot5hBsd81kMupNCXHaqbhv3huEbxAFMLnpcX2hniwn").unwrap(); let expected_hash = Hash::from_str("GKot5hBsd81kMupNCXHaqbhv3huEbxAFMLnpcX2hniwn").unwrap();
assert_eq!(result, (expected_hash, 0)); assert_eq!(result, (expected_hash, 0));
@ -7941,16 +7935,16 @@ pub mod tests {
let sum = raw_expected.iter().map(|item| item.lamports).sum(); let sum = raw_expected.iter().map(|item| item.lamports).sum();
let db = AccountsDb::new(Vec::new(), &ClusterType::Development); let db = AccountsDb::new(Vec::new(), &ClusterType::Development);
let result = db let result = db
.calculate_accounts_hash_without_index( .calculate_accounts_hash_without_index(&mut CalcAccountsHashConfig {
TempDir::new().unwrap().path(), accounts_hash_cache_path: TempDir::new().unwrap().path(),
&get_storage_refs(&storages), storages: &get_storage_refs(&storages),
None, thread_pool: None,
HashStats::default(), stats: HashStats::default(),
false, check_hash: false,
None, accounts_cache_and_ancestors: None,
None, filler_account_suffix: None,
None, num_hash_scan_passes: None,
) })
.unwrap(); .unwrap();
assert_eq!(result, (expected_hash, sum)); assert_eq!(result, (expected_hash, sum));

30
runtime/src/accounts_hash.rs
View File

@ -1,12 +1,16 @@
use { use {
crate::{
accounts_cache::AccountsCache, accounts_db::AccountInfoAccountsIndex, ancestors::Ancestors,
sorted_storages::SortedStorages,
},
log::*, log::*,
rayon::prelude::*, rayon::{prelude::*, ThreadPool},
solana_measure::measure::Measure, solana_measure::measure::Measure,
solana_sdk::{ solana_sdk::{
hash::{Hash, Hasher}, hash::{Hash, Hasher},
pubkey::Pubkey, pubkey::Pubkey,
}, },
std::{borrow::Borrow, convert::TryInto, sync::Mutex}, std::{borrow::Borrow, convert::TryInto, path::Path, sync::Mutex},
}; };
pub const ZERO_RAW_LAMPORTS_SENTINEL: u64 = std::u64::MAX; pub const ZERO_RAW_LAMPORTS_SENTINEL: u64 = std::u64::MAX;
pub const MERKLE_FANOUT: usize = 16; pub const MERKLE_FANOUT: usize = 16;
@ -18,6 +22,28 @@ pub struct PreviousPass {
pub lamports: u64, pub lamports: u64,
} }
/// parameters to calculate accounts hash
pub struct CalcAccountsHashConfig<'a> {
pub accounts_hash_cache_path: &'a Path,
pub storages: &'a SortedStorages<'a>,
pub thread_pool: Option<&'a ThreadPool>,
pub stats: HashStats,
pub check_hash: bool,
pub accounts_cache_and_ancestors: Option<(
&'a AccountsCache,
&'a Ancestors,
&'a AccountInfoAccountsIndex,
)>,
// these should be gone soon as we get an AccountsDb '&self'
pub filler_account_suffix: Option<&'a Pubkey>,
pub num_hash_scan_passes: Option<usize>,
// to come soon
/*
pub rent_collector: RentCollector,
pub epoch_schedule: EpochSchedule,
*/
}
#[derive(Debug, Default)] #[derive(Debug, Default)]
pub struct HashStats { pub struct HashStats {
pub scan_time_total_us: u64, pub scan_time_total_us: u64,