gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rename to ClusterType and restore devnet compat. (#12068)

Browse Source 
* Rename to ClusterType and restore devnet compat.

* De-duplicate parse code and add comments

* Adjust default Devnet genesis & reduce it in tests
This commit is contained in:
Ryo Onodera
2020-09-08 23:55:09 +09:00
committed by GitHub
parent c67f8bd821
commit 53b8ea4464
25 changed files with 376 additions and 319 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
runtime/src/accounts_db.rs
View File

@@ -33,7 +33,7 @@ use solana_rayon_threadlimit::get_thread_count;
use solana_sdk::{
account::Account,
clock::{Epoch, Slot},
genesis_config::OperatingMode,
genesis_config::ClusterType,
hash::{Hash, Hasher},
pubkey::Pubkey,
};
@@ -416,7 +416,7 @@ pub struct AccountsDB {
stats: AccountsStats,
operating_mode: Option<OperatingMode>,
cluster_type: Option<ClusterType>,
}
#[derive(Debug, Default)]
@@ -478,18 +478,18 @@ impl Default for AccountsDB {
frozen_accounts: HashMap::new(),
dead_slots: RwLock::new(HashSet::new()),
stats: AccountsStats::default(),
operating_mode: None,
cluster_type: None,
}
}
}
impl AccountsDB {
pub fn new(paths: Vec<PathBuf>, operating_mode: &OperatingMode) -> Self {
pub fn new(paths: Vec<PathBuf>, cluster_type: &ClusterType) -> Self {
let new = if !paths.is_empty() {
Self {
paths,
temp_paths: None,
operating_mode: Some(*operating_mode),
cluster_type: Some(*cluster_type),
..Self::default()
}
} else {
@@ -499,7 +499,7 @@ impl AccountsDB {
Self {
paths,
temp_paths: Some(temp_dirs),
operating_mode: Some(*operating_mode),
cluster_type: Some(*cluster_type),
..Self::default()
}
};
@@ -515,14 +515,14 @@ impl AccountsDB {
pub fn new_single() -> Self {
AccountsDB {
min_num_stores: 0,
..AccountsDB::new(Vec::new(), &OperatingMode::Development)
..AccountsDB::new(Vec::new(), &ClusterType::Development)
}
}
#[cfg(test)]
pub fn new_sized(paths: Vec<PathBuf>, file_size: u64) -> Self {
AccountsDB {
file_size,
..AccountsDB::new(paths, &OperatingMode::Development)
..AccountsDB::new(paths, &ClusterType::Development)
}
}
@@ -1280,12 +1280,24 @@ impl AccountsDB {
assert!(self.storage.read().unwrap().0.get(&remove_slot).is_none());
}
fn include_owner(cluster_type: &ClusterType, slot: Slot) -> bool {
// When devnet was moved to stable release channel, it was done without
// hashing account.owner. That's because devnet's slot was lower than
// 5_800_000 and the release channel's gating lacked ClusterType at the time...
match cluster_type {
ClusterType::Devnet => slot >= 5_800_000,
_ => true,
}
}
pub fn hash_stored_account(
slot: Slot,
account: &StoredAccount,
operating_mode: &OperatingMode,
cluster_type: &ClusterType,
) -> Hash {
if slot > Self::get_blake3_slot(operating_mode) {
let include_owner = Self::include_owner(cluster_type, slot);
if slot > Self::get_blake3_slot(cluster_type) {
Self::blake3_hash_account_data(
slot,
account.account_meta.lamports,
@@ -1294,6 +1306,7 @@ impl AccountsDB {
account.account_meta.rent_epoch,
account.data,
&account.meta.pubkey,
include_owner,
)
} else {
Self::hash_account_data(
@@ -1304,6 +1317,7 @@ impl AccountsDB {
account.account_meta.rent_epoch,
account.data,
&account.meta.pubkey,
include_owner,
)
}
}
@@ -1312,9 +1326,11 @@ impl AccountsDB {
slot: Slot,
account: &Account,
pubkey: &Pubkey,
operating_mode: &OperatingMode,
cluster_type: &ClusterType,
) -> Hash {
if slot > Self::get_blake3_slot(operating_mode) {
let include_owner = Self::include_owner(cluster_type, slot);
if slot > Self::get_blake3_slot(cluster_type) {
Self::blake3_hash_account_data(
slot,
account.lamports,
@@ -1323,6 +1339,7 @@ impl AccountsDB {
account.rent_epoch,
&account.data,
pubkey,
include_owner,
)
} else {
Self::hash_account_data(
@@ -1333,6 +1350,7 @@ impl AccountsDB {
account.rent_epoch,
&account.data,
pubkey,
include_owner,
)
}
}
@@ -1360,6 +1378,7 @@ impl AccountsDB {
rent_epoch: Epoch,
data: &[u8],
pubkey: &Pubkey,
include_owner: bool,
) -> Hash {
if lamports == 0 {
return Hash::default();
@@ -1381,7 +1400,9 @@ impl AccountsDB {
hasher.hash(&[0u8; 1]);
}
hasher.hash(&owner.as_ref());
if include_owner {
hasher.hash(&owner.as_ref());
}
hasher.hash(&pubkey.as_ref());
hasher.result()
@@ -1395,6 +1416,7 @@ impl AccountsDB {
rent_epoch: Epoch,
data: &[u8],
pubkey: &Pubkey,
include_owner: bool,
) -> Hash {
if lamports == 0 {
return Hash::default();
@@ -1416,20 +1438,23 @@ impl AccountsDB {
hasher.update(&[0u8; 1]);
}
hasher.update(&owner.as_ref());
if include_owner {
hasher.update(&owner.as_ref());
}
hasher.update(&pubkey.as_ref());
Hash(<[u8; solana_sdk::hash::HASH_BYTES]>::try_from(hasher.finalize().as_slice()).unwrap())
}
fn get_blake3_slot(operating_mode: &OperatingMode) -> Slot {
match operating_mode {
fn get_blake3_slot(cluster_type: &ClusterType) -> Slot {
match cluster_type {
ClusterType::Development => 0,
// Epoch 400
OperatingMode::Development => 3_276_800,
ClusterType::Devnet => 3_276_800,
// Epoch 78
OperatingMode::Stable => 33_696_000,
ClusterType::MainnetBeta => 33_696_000,
// Epoch 95
OperatingMode::Preview => 35_516_256,
ClusterType::Testnet => 35_516_256,
}
}
@@ -1652,8 +1677,8 @@ impl AccountsDB {
*slot,
&account,
&self
.operating_mode
.expect("Operating mode must be set at initialization"),
.cluster_type
.expect("Cluster type must be set at initialization"),
);
if hash != *account.hash {
mismatch_found.fetch_add(1, Ordering::Relaxed);
@@ -1893,14 +1918,14 @@ impl AccountsDB {
&self,
slot: Slot,
accounts: &[(&Pubkey, &Account)],
operating_mode: &OperatingMode,
cluster_type: &ClusterType,
) -> Vec<Hash> {
let mut stats = BankHashStats::default();
let hashes: Vec<_> = accounts
.iter()
.map(|(pubkey, account)| {
stats.update(account);
Self::hash_account(slot, account, pubkey, operating_mode)
Self::hash_account(slot, account, pubkey, cluster_type)
})
.collect();
@@ -1969,8 +1994,8 @@ impl AccountsDB {
slot,
accounts,
&self
.operating_mode
.expect("Operating mode must be set at initialization"),
.cluster_type
.expect("Cluster type must be set at initialization"),
);
self.store_with_hashes(slot, accounts, &hashes);
}
@@ -2182,7 +2207,7 @@ pub mod tests {
#[test]
fn test_accountsdb_add_root() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::default();
let account0 = Account::new(1, 0, &key);
@@ -2195,7 +2220,7 @@ pub mod tests {
#[test]
fn test_accountsdb_latest_ancestor() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::default();
let account0 = Account::new(1, 0, &key);
@@ -2222,7 +2247,7 @@ pub mod tests {
#[test]
fn test_accountsdb_latest_ancestor_with_root() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::default();
let account0 = Account::new(1, 0, &key);
@@ -2242,7 +2267,7 @@ pub mod tests {
#[test]
fn test_accountsdb_root_one_slot() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::default();
let account0 = Account::new(1, 0, &key);
@@ -2283,7 +2308,7 @@ pub mod tests {
#[test]
fn test_accountsdb_add_root_many() {
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&db, &mut pubkeys, 0, 100, 0, 0);
@@ -2372,7 +2397,7 @@ pub mod tests {
let key = Pubkey::default();
// 1 token in the "root", i.e. db zero
let db0 = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db0 = AccountsDB::new(Vec::new(), &ClusterType::Development);
let account0 = Account::new(1, 0, &key);
db0.store(0, &[(&key, &account0)]);
@@ -2391,7 +2416,7 @@ pub mod tests {
#[test]
fn test_remove_unrooted_slot() {
let unrooted_slot = 9;
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::default();
let account0 = Account::new(1, 0, &key);
let ancestors: HashMap<_, _> = vec![(unrooted_slot, 1)].into_iter().collect();
@@ -2437,7 +2462,7 @@ pub mod tests {
#[test]
fn test_remove_unrooted_slot_snapshot() {
let unrooted_slot = 9;
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::new_rand();
let account0 = Account::new(1, 0, &key);
db.store(unrooted_slot, &[(&key, &account0)]);
@@ -2564,7 +2589,7 @@ pub mod tests {
#[test]
fn test_account_one() {
let (_accounts_dirs, paths) = get_temp_accounts_paths(1).unwrap();
let db = AccountsDB::new(paths, &OperatingMode::Development);
let db = AccountsDB::new(paths, &ClusterType::Development);
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&db, &mut pubkeys, 0, 1, 0, 0);
let ancestors = vec![(0, 0)].into_iter().collect();
@@ -2577,7 +2602,7 @@ pub mod tests {
#[test]
fn test_account_many() {
let (_accounts_dirs, paths) = get_temp_accounts_paths(2).unwrap();
let db = AccountsDB::new(paths, &OperatingMode::Development);
let db = AccountsDB::new(paths, &ClusterType::Development);
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&db, &mut pubkeys, 0, 100, 0, 0);
check_accounts(&db, &pubkeys, 0, 100, 1);
@@ -2695,7 +2720,7 @@ pub mod tests {
#[test]
fn test_purge_slot_not_root() {
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let accounts = AccountsDB::new(Vec::new(), &ClusterType::Development);
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
let ancestors = vec![(0, 0)].into_iter().collect();
@@ -2706,7 +2731,7 @@ pub mod tests {
#[test]
fn test_purge_slot_after_root() {
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let accounts = AccountsDB::new(Vec::new(), &ClusterType::Development);
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
let ancestors = vec![(0, 0)].into_iter().collect();
@@ -2721,7 +2746,7 @@ pub mod tests {
//This test is pedantic
//A slot is purged when a non root bank is cleaned up. If a slot is behind root but it is
//not root, it means we are retaining dead banks.
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let accounts = AccountsDB::new(Vec::new(), &ClusterType::Development);
let pubkey = Pubkey::new_rand();
let account = Account::new(1, 0, &Account::default().owner);
//store an account
@@ -2799,7 +2824,7 @@ pub mod tests {
fn test_clean_old_with_normal_account() {
solana_logger::setup();
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let accounts = AccountsDB::new(Vec::new(), &ClusterType::Development);
let pubkey = Pubkey::new_rand();
let account = Account::new(1, 0, &Account::default().owner);
//store an account
@@ -2825,7 +2850,7 @@ pub mod tests {
fn test_clean_old_with_zero_lamport_account() {
solana_logger::setup();
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let accounts = AccountsDB::new(Vec::new(), &ClusterType::Development);
let pubkey1 = Pubkey::new_rand();
let pubkey2 = Pubkey::new_rand();
let normal_account = Account::new(1, 0, &Account::default().owner);
@@ -2855,7 +2880,7 @@ pub mod tests {
fn test_clean_old_with_both_normal_and_zero_lamport_accounts() {
solana_logger::setup();
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let accounts = AccountsDB::new(Vec::new(), &ClusterType::Development);
let pubkey1 = Pubkey::new_rand();
let pubkey2 = Pubkey::new_rand();
let normal_account = Account::new(1, 0, &Account::default().owner);
@@ -2888,7 +2913,7 @@ pub mod tests {
fn test_uncleaned_roots_with_account() {
solana_logger::setup();
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let accounts = AccountsDB::new(Vec::new(), &ClusterType::Development);
let pubkey = Pubkey::new_rand();
let account = Account::new(1, 0, &Account::default().owner);
//store an account
@@ -2908,7 +2933,7 @@ pub mod tests {
fn test_uncleaned_roots_with_no_account() {
solana_logger::setup();
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let accounts = AccountsDB::new(Vec::new(), &ClusterType::Development);
assert_eq!(accounts.uncleaned_root_count(), 0);
@@ -3342,7 +3367,7 @@ pub mod tests {
#[test]
fn test_accountsdb_scan_accounts() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::default();
let key0 = Pubkey::new_rand();
let account0 = Account::new(1, 0, &key);
@@ -3406,7 +3431,7 @@ pub mod tests {
#[test]
fn test_store_large_account() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::default();
let data_len = DEFAULT_FILE_SIZE as usize + 7;
@@ -3491,7 +3516,7 @@ pub mod tests {
fn test_frozen_account_lamport_increase() {
let frozen_pubkey =
Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap();
let mut db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let mut db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let mut account = Account::new(1, 42, &frozen_pubkey);
db.store(0, &[(&frozen_pubkey, &account)]);
@@ -3526,7 +3551,7 @@ pub mod tests {
fn test_frozen_account_lamport_decrease() {
let frozen_pubkey =
Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap();
let mut db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let mut db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let mut account = Account::new(1, 42, &frozen_pubkey);
db.store(0, &[(&frozen_pubkey, &account)]);
@@ -3546,7 +3571,7 @@ pub mod tests {
fn test_frozen_account_nonexistent() {
let frozen_pubkey =
Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap();
let mut db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let mut db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let ancestors = vec![(0, 0)].into_iter().collect();
db.freeze_accounts(&ancestors, &[frozen_pubkey]);
@@ -3559,7 +3584,7 @@ pub mod tests {
fn test_frozen_account_data_modified() {
let frozen_pubkey =
Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap();
let mut db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let mut db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let mut account = Account::new(1, 42, &frozen_pubkey);
db.store(0, &[(&frozen_pubkey, &account)]);
@@ -3612,7 +3637,7 @@ pub mod tests {
Hash::from_str("4StuvYHFd7xuShVXB94uHHvpqGMCaacdZnYB74QQkPA1").unwrap();
assert_eq!(
AccountsDB::hash_stored_account(slot, &stored_account, &OperatingMode::Development),
AccountsDB::hash_stored_account(slot, &stored_account, &ClusterType::Development),
expected_account_hash,
"StoredAccount's data layout might be changed; update hashing if needed."
);
@@ -3621,7 +3646,7 @@ pub mod tests {
slot,
&account,
&stored_account.meta.pubkey,
&OperatingMode::Development
&ClusterType::Development
),
expected_account_hash,
"Account-based hashing must be consistent with StoredAccount-based one."
@@ -3631,7 +3656,7 @@ pub mod tests {
#[test]
fn test_bank_hash_stats() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::default();
let some_data_len = 5;
@@ -3659,7 +3684,7 @@ pub mod tests {
fn test_verify_bank_hash() {
use BankHashVerificationError::*;
solana_logger::setup();
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::default();
let some_data_len = 0;
@@ -3697,7 +3722,7 @@ pub mod tests {
#[test]
fn test_verify_bank_hash_no_account() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let some_slot: Slot = 0;
let ancestors = vec![(some_slot, 0)].into_iter().collect();
@@ -3715,7 +3740,7 @@ pub mod tests {
fn test_verify_bank_hash_bad_account_hash() {
use BankHashVerificationError::*;
solana_logger::setup();
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::default();
let some_data_len = 0;
@@ -3725,7 +3750,7 @@ pub mod tests {
let accounts = &[(&key, &account)];
// update AccountsDB's bank hash but discard real account hashes
db.hash_accounts(some_slot, accounts, &OperatingMode::Development);
db.hash_accounts(some_slot, accounts, &ClusterType::Development);
// provide bogus account hashes
let some_hash = Hash::new(&[0xca; HASH_BYTES]);
db.store_with_hashes(some_slot, accounts, &[some_hash]);
@@ -3739,7 +3764,7 @@ pub mod tests {
#[test]
fn test_bad_bank_hash() {
use solana_sdk::signature::{Keypair, Signer};
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let some_slot: Slot = 0;
let ancestors: Ancestors = [(some_slot, 0)].iter().copied().collect();
@@ -3764,12 +3789,7 @@ pub mod tests {
for (key, account) in &accounts_keys {
assert_eq!(
db.load_account_hash(&ancestors, key),
AccountsDB::hash_account(
some_slot,
&account,
&key,
&OperatingMode::Development
)
AccountsDB::hash_account(some_slot, &account, &key, &ClusterType::Development)
);
}
}
@@ -3777,13 +3797,13 @@ pub mod tests {
#[test]
fn test_get_snapshot_storages_empty() {
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
assert!(db.get_snapshot_storages(0).is_empty());
}
#[test]
fn test_get_snapshot_storages_only_older_than_or_equal_to_snapshot_slot() {
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::default();
let account = Account::new(1, 0, &key);
@@ -3801,7 +3821,7 @@ pub mod tests {
#[test]
fn test_get_snapshot_storages_only_non_empty() {
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::default();
let account = Account::new(1, 0, &key);
@@ -3825,7 +3845,7 @@ pub mod tests {
#[test]
fn test_get_snapshot_storages_only_roots() {
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::default();
let account = Account::new(1, 0, &key);
@@ -3841,7 +3861,7 @@ pub mod tests {
#[test]
fn test_get_snapshot_storages_exclude_empty() {
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let db = AccountsDB::new(Vec::new(), &ClusterType::Development);
let key = Pubkey::default();
let account = Account::new(1, 0, &key);
@@ -3860,7 +3880,7 @@ pub mod tests {
#[test]
#[should_panic(expected = "double remove of account in slot: 0/store: 0!!")]
fn test_storage_remove_account_double_remove() {
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let accounts = AccountsDB::new(Vec::new(), &ClusterType::Development);
let pubkey = Pubkey::new_rand();
let account = Account::new(1, 0, &Account::default().owner);
accounts.store(0, &[(&pubkey, &account)]);