From 2b4af485370b88feabd66be63b09461ff4064ea2 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2020 20:47:25 +0000 Subject: [PATCH] Switch account hashing to blake3 (bp #11969) (#11991) * Switch account hashing to blake3 (#11969) * Switch account hashing to blake3 Co-authored-by: Carl (cherry picked from commit af08221aecc613eaf695a032a44ef66ed7fa9618) # Conflicts: # programs/bpf/Cargo.lock # runtime/src/accounts.rs # runtime/src/accounts_db.rs # runtime/src/serde_snapshot.rs * Resolve conflicts Co-authored-by: carllin Co-authored-by: Carl --- Cargo.lock | 47 ++++- accounts-bench/src/main.rs | 4 +- ledger/src/snapshot_utils.rs | 2 + programs/bpf/Cargo.lock | 81 +++++++- runtime/Cargo.toml | 1 + runtime/benches/accounts.rs | 26 ++- runtime/src/accounts.rs | 25 +-- runtime/src/accounts_db.rs | 286 ++++++++++++++++++++-------- runtime/src/bank.rs | 2 +- runtime/src/serde_snapshot.rs | 7 +- runtime/src/serde_snapshot/tests.rs | 6 +- sdk/src/hash.rs | 2 +- 12 files changed, 374 insertions(+), 115 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6a4ae927d5..abd2ebde01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -269,6 +269,21 @@ dependencies = [ "constant_time_eq", ] +[[package]] +name = "blake3" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce4f9586c9a3151c4b49b19e82ba163dd073614dd057e53c969e1a4db5b52720" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "crypto-mac 0.8.0", + "digest 0.9.0", +] + [[package]] name = "block-buffer" version = "0.7.3" @@ -667,6 +682,16 @@ dependencies = [ "subtle 1.0.0", ] +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array 0.14.3", + "subtle 2.2.3", +] + [[package]] name = "csv" version = "1.1.3" @@ -706,7 +731,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d85653f070353a16313d0046f173f70d1aadd5b42600a14de626f0dfb3473a5" dependencies = [ "byteorder", - "digest", + "digest 0.8.1", "rand_core 0.5.1", "subtle 2.2.3", "zeroize", @@ -749,6 +774,15 @@ dependencies = [ "generic-array 0.12.3", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.3", +] + [[package]] name = "dir-diff" version = "0.3.2" @@ -1323,8 +1357,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" dependencies = [ - "crypto-mac", - "digest", + "crypto-mac 0.7.0", + "digest 0.8.1", ] [[package]] @@ -2295,7 +2329,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" dependencies = [ "byteorder", - "crypto-mac", + "crypto-mac 0.7.0", ] [[package]] @@ -3133,7 +3167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" dependencies = [ "block-buffer", - "digest", + "digest 0.8.1", "fake-simd", "opaque-debug", ] @@ -3151,7 +3185,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" dependencies = [ "block-buffer", - "digest", + "digest 0.8.1", "fake-simd", "opaque-debug", ] @@ -4138,6 +4172,7 @@ version = "1.2.27" dependencies = [ "assert_matches", "bincode", + "blake3", "bv", "byteorder", "fnv", diff --git a/accounts-bench/src/main.rs b/accounts-bench/src/main.rs index 82dfbd3254..ff794f226b 100644 --- a/accounts-bench/src/main.rs +++ b/accounts-bench/src/main.rs @@ -5,7 +5,7 @@ use solana_runtime::{ accounts::{create_test_accounts, update_accounts, Accounts}, accounts_index::Ancestors, }; -use solana_sdk::pubkey::Pubkey; +use solana_sdk::{genesis_config::OperatingMode, pubkey::Pubkey}; use std::fs; use std::path::PathBuf; @@ -54,7 +54,7 @@ fn main() { if fs::remove_dir_all(path.clone()).is_err() { println!("Warning: Couldn't remove {:?}", path); } - let accounts = Accounts::new(vec![path]); + let accounts = Accounts::new(vec![path], OperatingMode::Preview); println!("Creating {} accounts", num_accounts); let mut create_time = Measure::start("create accounts"); let pubkeys: Vec<_> = (0..num_slots) diff --git a/ledger/src/snapshot_utils.rs b/ledger/src/snapshot_utils.rs index 3d06bfaac5..a05a474bf8 100644 --- a/ledger/src/snapshot_utils.rs +++ b/ledger/src/snapshot_utils.rs @@ -738,6 +738,7 @@ where bank.slot(), &mut stream, &append_vecs_path, + genesis_config.operating_mode, ), SnapshotVersion::V1_2_0 => bankrc_from_stream( SerdeStyle::NEWER, @@ -745,6 +746,7 @@ where bank.slot(), &mut stream, &append_vecs_path, + genesis_config.operating_mode, ), }?; Arc::get_mut(&mut Arc::get_mut(&mut bankrc.accounts).unwrap().accounts_db) diff --git a/programs/bpf/Cargo.lock b/programs/bpf/Cargo.lock index bc2e0a71ef..6a3a32b05a 100644 --- a/programs/bpf/Cargo.lock +++ b/programs/bpf/Cargo.lock @@ -24,6 +24,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" + [[package]] name = "ascii" version = "0.7.1" @@ -96,6 +108,21 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "blake3" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce4f9586c9a3151c4b49b19e82ba163dd073614dd057e53c969e1a4db5b52720" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "crypto-mac 0.8.0", + "digest 0.9.0", +] + [[package]] name = "block-buffer" version = "0.7.3" @@ -224,6 +251,21 @@ dependencies = [ "byteorder 1.3.4", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "crc32fast" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.4.3" @@ -292,6 +334,16 @@ dependencies = [ "subtle 1.0.0", ] +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array 0.14.3", + "subtle 2.2.2", +] + [[package]] name = "curve25519-dalek" version = "2.1.0" @@ -299,7 +351,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d85653f070353a16313d0046f173f70d1aadd5b42600a14de626f0dfb3473a5" dependencies = [ "byteorder 1.3.4", - "digest", + "digest 0.8.1", "rand_core", "subtle 2.2.3", "zeroize", @@ -314,6 +366,24 @@ dependencies = [ "generic-array 0.12.3", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.3", +] + +[[package]] +name = "dir-diff" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2860407d7d7e2e004bb2128510ad9e8d669e76fa005ccf567977b5d71b8b4a0b" +dependencies = [ + "walkdir", +] + [[package]] name = "dtoa" version = "0.4.6" @@ -635,8 +705,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" dependencies = [ - "crypto-mac", - "digest", + "crypto-mac 0.7.0", + "digest 0.8.1", ] [[package]] @@ -1108,7 +1178,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" dependencies = [ "byteorder 1.3.4", - "crypto-mac", + "crypto-mac 0.7.0", ] [[package]] @@ -1535,7 +1605,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" dependencies = [ "block-buffer", - "digest", + "digest 0.8.1", "fake-simd", "opaque-debug", ] @@ -1810,6 +1880,7 @@ name = "solana-runtime" version = "1.2.27" dependencies = [ "bincode", + "blake3", "bv", "byteorder 1.3.4", "fnv", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index c59f24a33a..56da6333e1 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -10,6 +10,7 @@ edition = "2018" [dependencies] bincode = "1.3.1" +blake3 = "0.3.6" bv = { version = "0.11.1", features = ["serde"] } byteorder = "1.3.4" fnv = "1.0.7" diff --git a/runtime/benches/accounts.rs b/runtime/benches/accounts.rs index 656b43f3bb..03c48f3b56 100644 --- a/runtime/benches/accounts.rs +++ b/runtime/benches/accounts.rs @@ -6,7 +6,11 @@ use solana_runtime::{ accounts::{create_test_accounts, Accounts}, bank::*, }; -use solana_sdk::{account::Account, genesis_config::create_genesis_config, pubkey::Pubkey}; +use solana_sdk::{ + account::Account, + genesis_config::{create_genesis_config, OperatingMode}, + pubkey::Pubkey, +}; use std::{path::PathBuf, sync::Arc}; use test::Bencher; @@ -65,7 +69,10 @@ fn test_accounts_squash(bencher: &mut Bencher) { #[bench] fn test_accounts_hash_bank_hash(bencher: &mut Bencher) { - let accounts = Accounts::new(vec![PathBuf::from("bench_accounts_hash_internal")]); + let accounts = Accounts::new( + vec![PathBuf::from("bench_accounts_hash_internal")], + OperatingMode::Development, + ); let mut pubkeys: Vec = vec![]; create_test_accounts(&accounts, &mut pubkeys, 60000, 0); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -76,7 +83,10 @@ fn test_accounts_hash_bank_hash(bencher: &mut Bencher) { #[bench] fn test_update_accounts_hash(bencher: &mut Bencher) { solana_logger::setup(); - let accounts = Accounts::new(vec![PathBuf::from("update_accounts_hash")]); + let accounts = Accounts::new( + vec![PathBuf::from("update_accounts_hash")], + OperatingMode::Development, + ); let mut pubkeys: Vec = vec![]; create_test_accounts(&accounts, &mut pubkeys, 50_000, 0); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -88,7 +98,10 @@ fn test_update_accounts_hash(bencher: &mut Bencher) { #[bench] fn test_accounts_delta_hash(bencher: &mut Bencher) { solana_logger::setup(); - let accounts = Accounts::new(vec![PathBuf::from("accounts_delta_hash")]); + let accounts = Accounts::new( + vec![PathBuf::from("accounts_delta_hash")], + OperatingMode::Development, + ); let mut pubkeys: Vec = vec![]; create_test_accounts(&accounts, &mut pubkeys, 100_000, 0); bencher.iter(|| { @@ -99,7 +112,10 @@ fn test_accounts_delta_hash(bencher: &mut Bencher) { #[bench] fn bench_delete_dependencies(bencher: &mut Bencher) { solana_logger::setup(); - let accounts = Accounts::new(vec![PathBuf::from("accounts_delete_deps")]); + let accounts = Accounts::new( + vec![PathBuf::from("accounts_delete_deps")], + OperatingMode::Development, + ); let mut old_pubkey = Pubkey::default(); let zero_account = Account::new(0, 0, &Account::default().owner); for i in 0..1000 { diff --git a/runtime/src/accounts.rs b/runtime/src/accounts.rs index 2b2b9a13ca..8488465133 100644 --- a/runtime/src/accounts.rs +++ b/runtime/src/accounts.rs @@ -18,6 +18,7 @@ use solana_sdk::{ account::Account, clock::Slot, fee_calculator::FeeCalculator, + genesis_config::OperatingMode, hash::Hash, message::Message, native_loader, nonce, @@ -75,8 +76,8 @@ impl Accounts { } } - pub fn new(paths: Vec) -> Self { - Self::new_with_frozen_accounts(paths, &HashMap::default(), &[]) + pub fn new(paths: Vec, operating_mode: OperatingMode) -> Self { + Self::new_with_frozen_accounts(paths, &HashMap::default(), &[], operating_mode) } pub fn new_from_parent(parent: &Accounts, slot: Slot, parent_slot: Slot) -> Self { @@ -94,10 +95,11 @@ impl Accounts { paths: Vec, ancestors: &Ancestors, frozen_account_pubkeys: &[Pubkey], + operating_mode: OperatingMode, ) -> Self { let mut accounts = Accounts { slot: 0, - accounts_db: Arc::new(AccountsDB::new(paths)), + accounts_db: Arc::new(AccountsDB::new(paths, operating_mode)), account_locks: Mutex::new(HashSet::new()), readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))), }; @@ -794,7 +796,6 @@ mod tests { account::Account, epoch_schedule::EpochSchedule, fee_calculator::FeeCalculator, - genesis_config::OperatingMode, hash::Hash, instruction::CompiledInstruction, message::Message, @@ -818,7 +819,7 @@ mod tests { ) -> Vec<(Result, Option)> { let mut hash_queue = BlockhashQueue::new(100); hash_queue.register_hash(&tx.message().recent_blockhash, &fee_calculator); - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), OperatingMode::Development); for ka in ka.iter() { accounts.store_slow(0, &ka.0, &ka.1); } @@ -1384,7 +1385,7 @@ mod tests { #[test] fn test_load_by_program_slot() { - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), OperatingMode::Development); // Load accounts owned by various programs into AccountsDB let pubkey0 = Pubkey::new_rand(); @@ -1407,7 +1408,7 @@ mod tests { #[test] fn test_accounts_account_not_found() { - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), OperatingMode::Development); let mut error_counters = ErrorCounters::default(); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -1429,7 +1430,7 @@ mod tests { #[test] #[should_panic] fn test_accounts_empty_bank_hash() { - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), OperatingMode::Development); accounts.bank_hash_at(1); } @@ -1445,7 +1446,7 @@ mod tests { let account2 = Account::new(3, 0, &Pubkey::default()); let account3 = Account::new(4, 0, &Pubkey::default()); - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), OperatingMode::Development); accounts.store_slow(0, &keypair0.pubkey(), &account0); accounts.store_slow(0, &keypair1.pubkey(), &account1); accounts.store_slow(0, &keypair2.pubkey(), &account2); @@ -1557,7 +1558,7 @@ mod tests { let account1 = Account::new(2, 0, &Pubkey::default()); let account2 = Account::new(3, 0, &Pubkey::default()); - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), OperatingMode::Development); accounts.store_slow(0, &keypair0.pubkey(), &account0); accounts.store_slow(0, &keypair1.pubkey(), &account1); accounts.store_slow(0, &keypair2.pubkey(), &account2); @@ -1687,7 +1688,7 @@ mod tests { let mut loaded = vec![loaded0, loaded1]; - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), OperatingMode::Development); { let mut readonly_locks = accounts.readonly_locks.write().unwrap(); let readonly_locks = readonly_locks.as_mut().unwrap(); @@ -1738,7 +1739,7 @@ mod tests { #[test] fn huge_clean() { solana_logger::setup(); - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), OperatingMode::Development); let mut old_pubkey = Pubkey::default(); let zero_account = Account::new(0, 0, &Account::default().owner); info!("storing.."); diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index 064d40a42b..b9ced042b0 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -22,7 +22,7 @@ use crate::{ accounts_index::{AccountsIndex, Ancestors, SlotList, SlotSlice}, append_vec::{AppendVec, StoredAccount, StoredMeta}, }; -use byteorder::{ByteOrder, LittleEndian}; +use blake3::traits::digest::Digest; use lazy_static::lazy_static; use log::*; use rand::{thread_rng, Rng}; @@ -33,9 +33,11 @@ use solana_rayon_threadlimit::get_thread_count; use solana_sdk::{ account::Account, clock::{Epoch, Slot}, + genesis_config::OperatingMode, hash::{Hash, Hasher}, pubkey::Pubkey, }; +use std::convert::TryFrom; use std::{ collections::{HashMap, HashSet}, io::{Error as IOError, Result as IOResult}, @@ -413,6 +415,8 @@ pub struct AccountsDB { dead_slots: RwLock>, stats: AccountsStats, + + operating_mode: Option, } #[derive(Debug, Default)] @@ -459,16 +463,18 @@ impl Default for AccountsDB { frozen_accounts: HashMap::new(), dead_slots: RwLock::new(HashSet::new()), stats: AccountsStats::default(), + operating_mode: None, } } } impl AccountsDB { - pub fn new(paths: Vec) -> Self { + pub fn new(paths: Vec, operating_mode: OperatingMode) -> Self { let new = if !paths.is_empty() { Self { paths, temp_paths: None, + operating_mode: Some(operating_mode), ..Self::default() } } else { @@ -478,6 +484,7 @@ impl AccountsDB { Self { paths, temp_paths: Some(temp_dirs), + operating_mode: Some(operating_mode), ..Self::default() } }; @@ -493,14 +500,14 @@ impl AccountsDB { pub fn new_single() -> Self { AccountsDB { min_num_stores: 0, - ..AccountsDB::new(Vec::new()) + ..AccountsDB::new(Vec::new(), OperatingMode::Development) } } #[cfg(test)] pub fn new_sized(paths: Vec, file_size: u64) -> Self { AccountsDB { file_size, - ..AccountsDB::new(paths) + ..AccountsDB::new(paths, OperatingMode::Development) } } @@ -1254,35 +1261,67 @@ impl AccountsDB { assert!(self.storage.read().unwrap().0.get(&remove_slot).is_none()); } - pub fn hash_stored_account(slot: Slot, account: &StoredAccount, include_owner: bool) -> Hash { - Self::hash_account_data( - slot, - account.account_meta.lamports, - &account.account_meta.owner, - account.account_meta.executable, - account.account_meta.rent_epoch, - account.data, - &account.meta.pubkey, - include_owner, - ) + pub fn hash_stored_account( + slot: Slot, + account: &StoredAccount, + operating_mode: OperatingMode, + include_owner: bool, + ) -> Hash { + if slot > Self::get_blake3_slot(operating_mode) { + Self::blake3_hash_account_data( + slot, + account.account_meta.lamports, + &account.account_meta.owner, + account.account_meta.executable, + account.account_meta.rent_epoch, + account.data, + &account.meta.pubkey, + include_owner, + ) + } else { + Self::hash_account_data( + slot, + account.account_meta.lamports, + &account.account_meta.owner, + account.account_meta.executable, + account.account_meta.rent_epoch, + account.data, + &account.meta.pubkey, + include_owner, + ) + } } pub fn hash_account( slot: Slot, account: &Account, pubkey: &Pubkey, + operating_mode: OperatingMode, include_owner: bool, ) -> Hash { - Self::hash_account_data( - slot, - account.lamports, - &account.owner, - account.executable, - account.rent_epoch, - &account.data, - pubkey, - include_owner, - ) + if slot > Self::get_blake3_slot(operating_mode) { + Self::blake3_hash_account_data( + slot, + account.lamports, + &account.owner, + account.executable, + account.rent_epoch, + &account.data, + pubkey, + include_owner, + ) + } else { + Self::hash_account_data( + slot, + account.lamports, + &account.owner, + account.executable, + account.rent_epoch, + &account.data, + pubkey, + include_owner, + ) + } } fn hash_frozen_account_data(account: &Account) -> Hash { @@ -1322,16 +1361,12 @@ impl AccountsDB { } let mut hasher = Hasher::default(); - let mut buf = [0u8; 8]; - LittleEndian::write_u64(&mut buf[..], lamports); - hasher.hash(&buf); + hasher.hash(&lamports.to_le_bytes()); - LittleEndian::write_u64(&mut buf[..], slot); - hasher.hash(&buf); + hasher.hash(&slot.to_le_bytes()); - LittleEndian::write_u64(&mut buf[..], rent_epoch); - hasher.hash(&buf); + hasher.hash(&rent_epoch.to_le_bytes()); hasher.hash(&data); @@ -1350,6 +1385,53 @@ impl AccountsDB { hasher.result() } + pub fn blake3_hash_account_data( + slot: Slot, + lamports: u64, + owner: &Pubkey, + executable: bool, + rent_epoch: Epoch, + data: &[u8], + pubkey: &Pubkey, + include_owner: bool, + ) -> Hash { + if lamports == 0 { + return Hash::default(); + } + + let mut hasher = blake3::Hasher::new(); + + hasher.update(&lamports.to_le_bytes()); + + hasher.update(&slot.to_le_bytes()); + + hasher.update(&rent_epoch.to_le_bytes()); + + hasher.update(&data); + + if executable { + hasher.update(&[1u8; 1]); + } else { + hasher.update(&[0u8; 1]); + } + + if include_owner { + hasher.update(&owner.as_ref()); + } + + hasher.update(&pubkey.as_ref()); + + Hash(<[u8; solana_sdk::hash::HASH_BYTES]>::try_from(hasher.finalize().as_slice()).unwrap()) + } + + fn get_blake3_slot(operating_mode: OperatingMode) -> Slot { + match operating_mode { + OperatingMode::Development => std::u64::MAX, + OperatingMode::Stable => std::u64::MAX, + OperatingMode::Preview => std::u64::MAX, + } + } + fn bulk_assign_write_version(&self, count: usize) -> u64 { self.write_version .fetch_add(count as u64, Ordering::Relaxed) @@ -1572,6 +1654,8 @@ impl AccountsDB { let hash = Self::hash_stored_account( *slot, &account, + self.operating_mode + .expect("Operating mode must be set at initialization"), Self::include_owner_in_hash(*slot), ); if hash != *account.hash { @@ -1808,13 +1892,24 @@ impl AccountsDB { } } - fn hash_accounts(&self, slot: Slot, accounts: &[(&Pubkey, &Account)]) -> Vec { + fn hash_accounts( + &self, + slot: Slot, + accounts: &[(&Pubkey, &Account)], + operating_mode: OperatingMode, + ) -> Vec { let mut stats = BankHashStats::default(); let hashes: Vec<_> = accounts .iter() .map(|(pubkey, account)| { stats.update(account); - Self::hash_account(slot, account, pubkey, Self::include_owner_in_hash(slot)) + Self::hash_account( + slot, + account, + pubkey, + operating_mode, + Self::include_owner_in_hash(slot), + ) }) .collect(); @@ -1879,7 +1974,12 @@ impl AccountsDB { /// Store the account update. pub fn store(&self, slot: Slot, accounts: &[(&Pubkey, &Account)]) { self.assert_frozen_accounts(accounts); - let hashes = self.hash_accounts(slot, accounts); + let hashes = self.hash_accounts( + slot, + accounts, + self.operating_mode + .expect("Operating mode must be set at initialization"), + ); self.store_with_hashes(slot, accounts, &hashes); } @@ -2042,7 +2142,7 @@ pub mod tests { #[test] fn test_accountsdb_add_root() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::default(); let account0 = Account::new(1, 0, &key); @@ -2055,7 +2155,7 @@ pub mod tests { #[test] fn test_accountsdb_latest_ancestor() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::default(); let account0 = Account::new(1, 0, &key); @@ -2082,7 +2182,7 @@ pub mod tests { #[test] fn test_accountsdb_latest_ancestor_with_root() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::default(); let account0 = Account::new(1, 0, &key); @@ -2102,7 +2202,7 @@ pub mod tests { #[test] fn test_accountsdb_root_one_slot() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::default(); let account0 = Account::new(1, 0, &key); @@ -2143,7 +2243,7 @@ pub mod tests { #[test] fn test_accountsdb_add_root_many() { - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let mut pubkeys: Vec = vec![]; create_account(&db, &mut pubkeys, 0, 100, 0, 0); @@ -2232,7 +2332,7 @@ pub mod tests { let key = Pubkey::default(); // 1 token in the "root", i.e. db zero - let db0 = AccountsDB::new(Vec::new()); + let db0 = AccountsDB::new(Vec::new(), OperatingMode::Development); let account0 = Account::new(1, 0, &key); db0.store(0, &[(&key, &account0)]); @@ -2251,7 +2351,7 @@ pub mod tests { #[test] fn test_remove_unrooted_slot() { let unrooted_slot = 9; - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::default(); let account0 = Account::new(1, 0, &key); let ancestors: HashMap<_, _> = vec![(unrooted_slot, 1)].into_iter().collect(); @@ -2297,7 +2397,7 @@ pub mod tests { #[test] fn test_remove_unrooted_slot_snapshot() { let unrooted_slot = 9; - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::new_rand(); let account0 = Account::new(1, 0, &key); db.store(unrooted_slot, &[(&key, &account0)]); @@ -2424,7 +2524,7 @@ pub mod tests { #[test] fn test_account_one() { let (_accounts_dirs, paths) = get_temp_accounts_paths(1).unwrap(); - let db = AccountsDB::new(paths); + let db = AccountsDB::new(paths, OperatingMode::Development); let mut pubkeys: Vec = vec![]; create_account(&db, &mut pubkeys, 0, 1, 0, 0); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -2437,7 +2537,7 @@ pub mod tests { #[test] fn test_account_many() { let (_accounts_dirs, paths) = get_temp_accounts_paths(2).unwrap(); - let db = AccountsDB::new(paths); + let db = AccountsDB::new(paths, OperatingMode::Development); let mut pubkeys: Vec = vec![]; create_account(&db, &mut pubkeys, 0, 100, 0, 0); check_accounts(&db, &pubkeys, 0, 100, 1); @@ -2555,7 +2655,7 @@ pub mod tests { #[test] fn test_purge_slot_not_root() { - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development); let mut pubkeys: Vec = vec![]; create_account(&accounts, &mut pubkeys, 0, 1, 0, 0); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -2566,7 +2666,7 @@ pub mod tests { #[test] fn test_purge_slot_after_root() { - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development); let mut pubkeys: Vec = vec![]; create_account(&accounts, &mut pubkeys, 0, 1, 0, 0); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -2581,7 +2681,7 @@ pub mod tests { //This test is pedantic //A slot is purged when a non root bank is cleaned up. If a slot is behind root but it is //not root, it means we are retaining dead banks. - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development); let pubkey = Pubkey::new_rand(); let account = Account::new(1, 0, &Account::default().owner); //store an account @@ -2659,7 +2759,7 @@ pub mod tests { fn test_clean_old_with_normal_account() { solana_logger::setup(); - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development); let pubkey = Pubkey::new_rand(); let account = Account::new(1, 0, &Account::default().owner); //store an account @@ -2685,7 +2785,7 @@ pub mod tests { fn test_clean_old_with_zero_lamport_account() { solana_logger::setup(); - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development); let pubkey1 = Pubkey::new_rand(); let pubkey2 = Pubkey::new_rand(); let normal_account = Account::new(1, 0, &Account::default().owner); @@ -2715,7 +2815,7 @@ pub mod tests { fn test_clean_old_with_both_normal_and_zero_lamport_accounts() { solana_logger::setup(); - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development); let pubkey1 = Pubkey::new_rand(); let pubkey2 = Pubkey::new_rand(); let normal_account = Account::new(1, 0, &Account::default().owner); @@ -2748,7 +2848,7 @@ pub mod tests { fn test_uncleaned_roots_with_account() { solana_logger::setup(); - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development); let pubkey = Pubkey::new_rand(); let account = Account::new(1, 0, &Account::default().owner); //store an account @@ -2768,7 +2868,7 @@ pub mod tests { fn test_uncleaned_roots_with_no_account() { solana_logger::setup(); - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development); assert_eq!(accounts.uncleaned_root_count(), 0); @@ -3246,7 +3346,7 @@ pub mod tests { #[test] fn test_accountsdb_scan_accounts() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::default(); let key0 = Pubkey::new_rand(); let account0 = Account::new(1, 0, &key); @@ -3310,7 +3410,7 @@ pub mod tests { #[test] fn test_store_large_account() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::default(); let data_len = DEFAULT_FILE_SIZE as usize + 7; @@ -3395,7 +3495,7 @@ pub mod tests { fn test_frozen_account_lamport_increase() { let frozen_pubkey = Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap(); - let mut db = AccountsDB::new(Vec::new()); + let mut db = AccountsDB::new(Vec::new(), OperatingMode::Development); let mut account = Account::new(1, 42, &frozen_pubkey); db.store(0, &[(&frozen_pubkey, &account)]); @@ -3430,7 +3530,7 @@ pub mod tests { fn test_frozen_account_lamport_decrease() { let frozen_pubkey = Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap(); - let mut db = AccountsDB::new(Vec::new()); + let mut db = AccountsDB::new(Vec::new(), OperatingMode::Development); let mut account = Account::new(1, 42, &frozen_pubkey); db.store(0, &[(&frozen_pubkey, &account)]); @@ -3450,7 +3550,7 @@ pub mod tests { fn test_frozen_account_nonexistent() { let frozen_pubkey = Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap(); - let mut db = AccountsDB::new(Vec::new()); + let mut db = AccountsDB::new(Vec::new(), OperatingMode::Development); let ancestors = vec![(0, 0)].into_iter().collect(); db.freeze_accounts(&ancestors, &[frozen_pubkey]); @@ -3463,7 +3563,7 @@ pub mod tests { fn test_frozen_account_data_modified() { let frozen_pubkey = Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap(); - let mut db = AccountsDB::new(Vec::new()); + let mut db = AccountsDB::new(Vec::new(), OperatingMode::Development); let mut account = Account::new(1, 42, &frozen_pubkey); db.store(0, &[(&frozen_pubkey, &account)]); @@ -3512,17 +3612,28 @@ pub mod tests { hash: &hash, }; let account = stored_account.clone_account(); - let expected_account_hash_without_owner = + let expected_account_hash = Hash::from_str("GGTsxvxwnMsNfN6yYbBVQaRgvbVLfxeWnGXNyB8iXDyE").unwrap(); assert_eq!( - AccountsDB::hash_stored_account(slot, &stored_account, false), - expected_account_hash_without_owner, + AccountsDB::hash_stored_account( + slot, + &stored_account, + OperatingMode::Development, + false + ), + expected_account_hash, "StoredAccount's data layout might be changed; update hashing if needed." ); assert_eq!( - AccountsDB::hash_account(slot, &account, &stored_account.meta.pubkey, false), - expected_account_hash_without_owner, + AccountsDB::hash_account( + slot, + &account, + &stored_account.meta.pubkey, + OperatingMode::Development, + false, + ), + expected_account_hash, "Account-based hashing must be consistent with StoredAccount-based one." ); @@ -3530,12 +3641,23 @@ pub mod tests { Hash::from_str("5iRNZVcAnq9JLYjSF2ibFhGEeq48r9Eq9HXxwm3BxywN").unwrap(); assert_eq!( - AccountsDB::hash_stored_account(slot, &stored_account, true), + AccountsDB::hash_stored_account( + slot, + &stored_account, + OperatingMode::Development, + true + ), expected_account_hash_with_owner, "StoredAccount's data layout might be changed; update hashing if needed (with owner)." ); assert_eq!( - AccountsDB::hash_account(slot, &account, &stored_account.meta.pubkey, true), + AccountsDB::hash_account( + slot, + &account, + &stored_account.meta.pubkey, + OperatingMode::Development, + true + ), expected_account_hash_with_owner, "Account-based hashing must be consistent with StoredAccount-based one (with owner)." ); @@ -3544,7 +3666,7 @@ pub mod tests { #[test] fn test_bank_hash_stats() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::default(); let some_data_len = 5; @@ -3572,7 +3694,7 @@ pub mod tests { fn test_verify_bank_hash() { use BankHashVerificationError::*; solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::default(); let some_data_len = 0; @@ -3610,7 +3732,7 @@ pub mod tests { #[test] fn test_verify_bank_hash_no_account() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let some_slot: Slot = 0; let ancestors = vec![(some_slot, 0)].into_iter().collect(); @@ -3628,7 +3750,7 @@ pub mod tests { fn test_verify_bank_hash_bad_account_hash() { use BankHashVerificationError::*; solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::default(); let some_data_len = 0; @@ -3638,7 +3760,7 @@ pub mod tests { let accounts = &[(&key, &account)]; // update AccountsDB's bank hash but discard real account hashes - db.hash_accounts(some_slot, accounts); + db.hash_accounts(some_slot, accounts, OperatingMode::Development); // provide bogus account hashes let some_hash = Hash::new(&[0xca; HASH_BYTES]); db.store_with_hashes(some_slot, accounts, &[some_hash]); @@ -3652,7 +3774,7 @@ pub mod tests { #[test] fn test_bad_bank_hash() { use solana_sdk::signature::{Keypair, Signer}; - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let some_slot: Slot = 0; let ancestors: Ancestors = [(some_slot, 0)].iter().copied().collect(); @@ -3677,7 +3799,13 @@ pub mod tests { for (key, account) in &accounts_keys { assert_eq!( db.load_account_hash(&ancestors, key), - AccountsDB::hash_account(some_slot, &account, &key, false) + AccountsDB::hash_account( + some_slot, + &account, + &key, + OperatingMode::Development, + false + ) ); } } @@ -3685,13 +3813,13 @@ pub mod tests { #[test] fn test_get_snapshot_storages_empty() { - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); assert!(db.get_snapshot_storages(0).is_empty()); } #[test] fn test_get_snapshot_storages_only_older_than_or_equal_to_snapshot_slot() { - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::default(); let account = Account::new(1, 0, &key); @@ -3709,7 +3837,7 @@ pub mod tests { #[test] fn test_get_snapshot_storages_only_non_empty() { - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::default(); let account = Account::new(1, 0, &key); @@ -3733,7 +3861,7 @@ pub mod tests { #[test] fn test_get_snapshot_storages_only_roots() { - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::default(); let account = Account::new(1, 0, &key); @@ -3749,7 +3877,7 @@ pub mod tests { #[test] fn test_get_snapshot_storages_exclude_empty() { - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), OperatingMode::Development); let key = Pubkey::default(); let account = Account::new(1, 0, &key); @@ -3768,7 +3896,7 @@ pub mod tests { #[test] #[should_panic(expected = "double remove of account in slot: 0/store: 0!!")] fn test_storage_remove_account_double_remove() { - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development); let pubkey = Pubkey::new_rand(); let account = Account::new(1, 0, &Account::default().owner); accounts.store(0, &[(&pubkey, &account)]); diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 34008c02ca..c75068d976 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -375,7 +375,7 @@ impl Bank { bank.operating_mode = Some(genesis_config.operating_mode); bank.ancestors.insert(bank.slot(), 0); - bank.rc.accounts = Arc::new(Accounts::new(paths)); + bank.rc.accounts = Arc::new(Accounts::new(paths, genesis_config.operating_mode)); bank.process_genesis_config(genesis_config); bank.finish_init(); diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index c856a93773..a1db318891 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -15,7 +15,7 @@ use { de::{DeserializeOwned, Visitor}, Deserialize, Deserializer, Serialize, Serializer, }, - solana_sdk::clock::Slot, + solana_sdk::{clock::Slot, genesis_config::OperatingMode}, std::{ cmp::min, collections::HashMap, @@ -104,6 +104,7 @@ pub fn bankrc_from_stream( slot: Slot, stream: &mut BufReader, stream_append_vecs_path: P, + operating_mode: OperatingMode, ) -> std::result::Result where R: Read, @@ -116,6 +117,7 @@ where $x::deserialize_accounts_db_fields(stream)?, account_paths, stream_append_vecs_path, + operating_mode, )?), slot, )) @@ -190,12 +192,13 @@ fn context_accountsdb_from_fields<'a, C, P>( account_db_fields: AccountDBFields, account_paths: &[PathBuf], stream_append_vecs_path: P, + operating_mode: OperatingMode, ) -> Result where C: TypeContext<'a>, P: AsRef, { - let accounts_db = AccountsDB::new(account_paths.to_vec()); + let accounts_db = AccountsDB::new(account_paths.to_vec(), operating_mode); let AccountDBFields(storage, version, slot, bank_hash_info) = account_db_fields; diff --git a/runtime/src/serde_snapshot/tests.rs b/runtime/src/serde_snapshot/tests.rs index 21aebf8ace..3676382240 100644 --- a/runtime/src/serde_snapshot/tests.rs +++ b/runtime/src/serde_snapshot/tests.rs @@ -10,7 +10,7 @@ use { solana_sdk::{ account::Account, clock::Slot, - genesis_config::create_genesis_config, + genesis_config::{create_genesis_config, OperatingMode}, pubkey::Pubkey, signature::{Keypair, Signer}, }, @@ -68,6 +68,7 @@ where C::deserialize_accounts_db_fields(stream)?, account_paths, stream_append_vecs_path, + OperatingMode::Development, ) } @@ -135,7 +136,7 @@ where fn test_accounts_serialize_style(serde_style: SerdeStyle) { solana_logger::setup(); let (_accounts_dir, paths) = get_temp_accounts_paths(4).unwrap(); - let accounts = Accounts::new(paths); + let accounts = Accounts::new(paths, OperatingMode::Development); let mut pubkeys: Vec = vec![]; create_test_accounts(&accounts, &mut pubkeys, 100, 0); @@ -227,6 +228,7 @@ fn test_bank_serialize_style(serde_style: SerdeStyle) { dbank.slot(), &mut reader, copied_accounts.path(), + OperatingMode::Development, ) .unwrap(), ref_sc, diff --git a/sdk/src/hash.rs b/sdk/src/hash.rs index 46f1a371bb..926e9b1855 100644 --- a/sdk/src/hash.rs +++ b/sdk/src/hash.rs @@ -8,7 +8,7 @@ use thiserror::Error; pub const HASH_BYTES: usize = 32; #[derive(Serialize, Deserialize, Clone, Copy, Default, Eq, PartialEq, Ord, PartialOrd, Hash)] #[repr(transparent)] -pub struct Hash([u8; HASH_BYTES]); +pub struct Hash(pub [u8; HASH_BYTES]); #[derive(Clone, Default)] pub struct Hasher {