gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Switch account hashing to blake3 (bp #11969) (#11991)

Browse Source 
* Switch account hashing to blake3 (#11969)

* Switch account hashing to blake3

Co-authored-by: Carl <carl@solana.com>
(cherry picked from commit af08221aec)

# Conflicts:
#	programs/bpf/Cargo.lock
#	runtime/src/accounts.rs
#	runtime/src/accounts_db.rs
#	runtime/src/serde_snapshot.rs

* Resolve conflicts

Co-authored-by: carllin <wumu727@gmail.com>
Co-authored-by: Carl <carl@solana.com>
This commit is contained in:
mergify[bot]
2020-09-04 20:47:25 +00:00
committed by GitHub
parent 2a842408bd
commit 2b4af48537
12 changed files with 374 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
Cargo.lock generated
View File

@ -269,6 +269,21 @@ dependencies = [
"constant_time_eq", "constant_time_eq",
] ]
[[package]]
name = "blake3"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce4f9586c9a3151c4b49b19e82ba163dd073614dd057e53c969e1a4db5b52720"
dependencies = [
"arrayref",
"arrayvec",
"cc",
"cfg-if",
"constant_time_eq",
"crypto-mac 0.8.0",
"digest 0.9.0",
]
[[package]] [[package]]
name = "block-buffer" name = "block-buffer"
version = "0.7.3" version = "0.7.3"
@ -667,6 +682,16 @@ dependencies = [
"subtle 1.0.0", "subtle 1.0.0",
] ]
[[package]]
name = "crypto-mac"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab"
dependencies = [
"generic-array 0.14.3",
"subtle 2.2.3",
]
[[package]] [[package]]
name = "csv" name = "csv"
version = "1.1.3" version = "1.1.3"
@ -706,7 +731,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d85653f070353a16313d0046f173f70d1aadd5b42600a14de626f0dfb3473a5" checksum = "5d85653f070353a16313d0046f173f70d1aadd5b42600a14de626f0dfb3473a5"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"digest", "digest 0.8.1",
"rand_core 0.5.1", "rand_core 0.5.1",
"subtle 2.2.3", "subtle 2.2.3",
"zeroize", "zeroize",
@ -749,6 +774,15 @@ dependencies = [
"generic-array 0.12.3", "generic-array 0.12.3",
] ]
[[package]]
name = "digest"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
dependencies = [
"generic-array 0.14.3",
]
[[package]] [[package]]
name = "dir-diff" name = "dir-diff"
version = "0.3.2" version = "0.3.2"
@ -1323,8 +1357,8 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695"
dependencies = [ dependencies = [
"crypto-mac", "crypto-mac 0.7.0",
"digest", "digest 0.8.1",
] ]
[[package]] [[package]]
@ -2295,7 +2329,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"crypto-mac", "crypto-mac 0.7.0",
] ]
[[package]] [[package]]
@ -3133,7 +3167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df"
dependencies = [ dependencies = [
"block-buffer", "block-buffer",
"digest", "digest 0.8.1",
"fake-simd", "fake-simd",
"opaque-debug", "opaque-debug",
] ]
@ -3151,7 +3185,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69"
dependencies = [ dependencies = [
"block-buffer", "block-buffer",
"digest", "digest 0.8.1",
"fake-simd", "fake-simd",
"opaque-debug", "opaque-debug",
] ]
@ -4138,6 +4172,7 @@ version = "1.2.27"
dependencies = [ dependencies = [
"assert_matches", "assert_matches",
"bincode", "bincode",
"blake3",
"bv", "bv",
"byteorder", "byteorder",
"fnv", "fnv",

4
accounts-bench/src/main.rs
View File

@ -5,7 +5,7 @@ use solana_runtime::{
accounts::{create_test_accounts, update_accounts, Accounts}, accounts::{create_test_accounts, update_accounts, Accounts},
accounts_index::Ancestors, accounts_index::Ancestors,
}; };
use solana_sdk::pubkey::Pubkey; use solana_sdk::{genesis_config::OperatingMode, pubkey::Pubkey};
use std::fs; use std::fs;
use std::path::PathBuf; use std::path::PathBuf;
@ -54,7 +54,7 @@ fn main() {
if fs::remove_dir_all(path.clone()).is_err() { if fs::remove_dir_all(path.clone()).is_err() {
println!("Warning: Couldn't remove {:?}", path); println!("Warning: Couldn't remove {:?}", path);
} }
let accounts = Accounts::new(vec![path]); let accounts = Accounts::new(vec![path], OperatingMode::Preview);
println!("Creating {} accounts", num_accounts); println!("Creating {} accounts", num_accounts);
let mut create_time = Measure::start("create accounts"); let mut create_time = Measure::start("create accounts");
let pubkeys: Vec<_> = (0..num_slots) let pubkeys: Vec<_> = (0..num_slots)

2
ledger/src/snapshot_utils.rs
View File

@ -738,6 +738,7 @@ where
bank.slot(), bank.slot(),
&mut stream, &mut stream,
&append_vecs_path, &append_vecs_path,
genesis_config.operating_mode,
), ),
SnapshotVersion::V1_2_0 => bankrc_from_stream( SnapshotVersion::V1_2_0 => bankrc_from_stream(
SerdeStyle::NEWER, SerdeStyle::NEWER,
@ -745,6 +746,7 @@ where
bank.slot(), bank.slot(),
&mut stream, &mut stream,
&append_vecs_path, &append_vecs_path,
genesis_config.operating_mode,
), ),
}?; }?;
Arc::get_mut(&mut Arc::get_mut(&mut bankrc.accounts).unwrap().accounts_db) Arc::get_mut(&mut Arc::get_mut(&mut bankrc.accounts).unwrap().accounts_db)

81
programs/bpf/Cargo.lock generated
View File

@ -24,6 +24,18 @@ dependencies = [
"memchr", "memchr",
] ]
[[package]]
name = "arrayref"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
[[package]]
name = "arrayvec"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8"
[[package]] [[package]]
name = "ascii" name = "ascii"
version = "0.7.1" version = "0.7.1"
@ -96,6 +108,21 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "blake3"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce4f9586c9a3151c4b49b19e82ba163dd073614dd057e53c969e1a4db5b52720"
dependencies = [
"arrayref",
"arrayvec",
"cc",
"cfg-if",
"constant_time_eq",
"crypto-mac 0.8.0",
"digest 0.9.0",
]
[[package]] [[package]]
name = "block-buffer" name = "block-buffer"
version = "0.7.3" version = "0.7.3"
@ -224,6 +251,21 @@ dependencies = [
"byteorder 1.3.4", "byteorder 1.3.4",
] ]
[[package]]
name = "constant_time_eq"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
[[package]]
name = "crc32fast"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
dependencies = [
"cfg-if",
]
[[package]] [[package]]
name = "crossbeam-channel" name = "crossbeam-channel"
version = "0.4.3" version = "0.4.3"
@ -292,6 +334,16 @@ dependencies = [
"subtle 1.0.0", "subtle 1.0.0",
] ]
[[package]]
name = "crypto-mac"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab"
dependencies = [
"generic-array 0.14.3",
"subtle 2.2.2",
]
[[package]] [[package]]
name = "curve25519-dalek" name = "curve25519-dalek"
version = "2.1.0" version = "2.1.0"
@ -299,7 +351,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d85653f070353a16313d0046f173f70d1aadd5b42600a14de626f0dfb3473a5" checksum = "5d85653f070353a16313d0046f173f70d1aadd5b42600a14de626f0dfb3473a5"
dependencies = [ dependencies = [
"byteorder 1.3.4", "byteorder 1.3.4",
"digest", "digest 0.8.1",
"rand_core", "rand_core",
"subtle 2.2.3", "subtle 2.2.3",
"zeroize", "zeroize",
@ -314,6 +366,24 @@ dependencies = [
"generic-array 0.12.3", "generic-array 0.12.3",
] ]
[[package]]
name = "digest"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
dependencies = [
"generic-array 0.14.3",
]
[[package]]
name = "dir-diff"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2860407d7d7e2e004bb2128510ad9e8d669e76fa005ccf567977b5d71b8b4a0b"
dependencies = [
"walkdir",
]
[[package]] [[package]]
name = "dtoa" name = "dtoa"
version = "0.4.6" version = "0.4.6"
@ -635,8 +705,8 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695"
dependencies = [ dependencies = [
"crypto-mac", "crypto-mac 0.7.0",
"digest", "digest 0.8.1",
] ]
[[package]] [[package]]
@ -1108,7 +1178,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9"
dependencies = [ dependencies = [
"byteorder 1.3.4", "byteorder 1.3.4",
"crypto-mac", "crypto-mac 0.7.0",
] ]
[[package]] [[package]]
@ -1535,7 +1605,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69"
dependencies = [ dependencies = [
"block-buffer", "block-buffer",
"digest", "digest 0.8.1",
"fake-simd", "fake-simd",
"opaque-debug", "opaque-debug",
] ]
@ -1810,6 +1880,7 @@ name = "solana-runtime"
version = "1.2.27" version = "1.2.27"
dependencies = [ dependencies = [
"bincode", "bincode",
"blake3",
"bv", "bv",
"byteorder 1.3.4", "byteorder 1.3.4",
"fnv", "fnv",

1
runtime/Cargo.toml
View File

@ -10,6 +10,7 @@ edition = "2018"
[dependencies] [dependencies]
bincode = "1.3.1" bincode = "1.3.1"
blake3 = "0.3.6"
bv = { version = "0.11.1", features = ["serde"] } bv = { version = "0.11.1", features = ["serde"] }
byteorder = "1.3.4" byteorder = "1.3.4"
fnv = "1.0.7" fnv = "1.0.7"

26
runtime/benches/accounts.rs
View File

@ -6,7 +6,11 @@ use solana_runtime::{
accounts::{create_test_accounts, Accounts}, accounts::{create_test_accounts, Accounts},
bank::*, bank::*,
}; };
use solana_sdk::{account::Account, genesis_config::create_genesis_config, pubkey::Pubkey}; use solana_sdk::{
account::Account,
genesis_config::{create_genesis_config, OperatingMode},
pubkey::Pubkey,
};
use std::{path::PathBuf, sync::Arc}; use std::{path::PathBuf, sync::Arc};
use test::Bencher; use test::Bencher;
@ -65,7 +69,10 @@ fn test_accounts_squash(bencher: &mut Bencher) {
#[bench] #[bench]
fn test_accounts_hash_bank_hash(bencher: &mut Bencher) { fn test_accounts_hash_bank_hash(bencher: &mut Bencher) {
let accounts = Accounts::new(vec![PathBuf::from("bench_accounts_hash_internal")]); let accounts = Accounts::new(
vec![PathBuf::from("bench_accounts_hash_internal")],
OperatingMode::Development,
);
let mut pubkeys: Vec<Pubkey> = vec![]; let mut pubkeys: Vec<Pubkey> = vec![];
create_test_accounts(&accounts, &mut pubkeys, 60000, 0); create_test_accounts(&accounts, &mut pubkeys, 60000, 0);
let ancestors = vec![(0, 0)].into_iter().collect(); let ancestors = vec![(0, 0)].into_iter().collect();
@ -76,7 +83,10 @@ fn test_accounts_hash_bank_hash(bencher: &mut Bencher) {
#[bench] #[bench]
fn test_update_accounts_hash(bencher: &mut Bencher) { fn test_update_accounts_hash(bencher: &mut Bencher) {
solana_logger::setup(); solana_logger::setup();
let accounts = Accounts::new(vec![PathBuf::from("update_accounts_hash")]); let accounts = Accounts::new(
vec![PathBuf::from("update_accounts_hash")],
OperatingMode::Development,
);
let mut pubkeys: Vec<Pubkey> = vec![]; let mut pubkeys: Vec<Pubkey> = vec![];
create_test_accounts(&accounts, &mut pubkeys, 50_000, 0); create_test_accounts(&accounts, &mut pubkeys, 50_000, 0);
let ancestors = vec![(0, 0)].into_iter().collect(); let ancestors = vec![(0, 0)].into_iter().collect();
@ -88,7 +98,10 @@ fn test_update_accounts_hash(bencher: &mut Bencher) {
#[bench] #[bench]
fn test_accounts_delta_hash(bencher: &mut Bencher) { fn test_accounts_delta_hash(bencher: &mut Bencher) {
solana_logger::setup(); solana_logger::setup();
let accounts = Accounts::new(vec![PathBuf::from("accounts_delta_hash")]); let accounts = Accounts::new(
vec![PathBuf::from("accounts_delta_hash")],
OperatingMode::Development,
);
let mut pubkeys: Vec<Pubkey> = vec![]; let mut pubkeys: Vec<Pubkey> = vec![];
create_test_accounts(&accounts, &mut pubkeys, 100_000, 0); create_test_accounts(&accounts, &mut pubkeys, 100_000, 0);
bencher.iter(|| { bencher.iter(|| {
@ -99,7 +112,10 @@ fn test_accounts_delta_hash(bencher: &mut Bencher) {
#[bench] #[bench]
fn bench_delete_dependencies(bencher: &mut Bencher) { fn bench_delete_dependencies(bencher: &mut Bencher) {
solana_logger::setup(); solana_logger::setup();
let accounts = Accounts::new(vec![PathBuf::from("accounts_delete_deps")]); let accounts = Accounts::new(
vec![PathBuf::from("accounts_delete_deps")],
OperatingMode::Development,
);
let mut old_pubkey = Pubkey::default(); let mut old_pubkey = Pubkey::default();
let zero_account = Account::new(0, 0, &Account::default().owner); let zero_account = Account::new(0, 0, &Account::default().owner);
for i in 0..1000 { for i in 0..1000 {

25
runtime/src/accounts.rs
View File

@ -18,6 +18,7 @@ use solana_sdk::{
account::Account, account::Account,
clock::Slot, clock::Slot,
fee_calculator::FeeCalculator, fee_calculator::FeeCalculator,
genesis_config::OperatingMode,
hash::Hash, hash::Hash,
message::Message, message::Message,
native_loader, nonce, native_loader, nonce,
@ -75,8 +76,8 @@ impl Accounts {
} }
} }
pub fn new(paths: Vec<PathBuf>) -> Self { pub fn new(paths: Vec<PathBuf>, operating_mode: OperatingMode) -> Self {
Self::new_with_frozen_accounts(paths, &HashMap::default(), &[]) Self::new_with_frozen_accounts(paths, &HashMap::default(), &[], operating_mode)
} }
pub fn new_from_parent(parent: &Accounts, slot: Slot, parent_slot: Slot) -> Self { pub fn new_from_parent(parent: &Accounts, slot: Slot, parent_slot: Slot) -> Self {
@ -94,10 +95,11 @@ impl Accounts {
paths: Vec<PathBuf>, paths: Vec<PathBuf>,
ancestors: &Ancestors, ancestors: &Ancestors,
frozen_account_pubkeys: &[Pubkey], frozen_account_pubkeys: &[Pubkey],
operating_mode: OperatingMode,
) -> Self { ) -> Self {
let mut accounts = Accounts { let mut accounts = Accounts {
slot: 0, slot: 0,
accounts_db: Arc::new(AccountsDB::new(paths)), accounts_db: Arc::new(AccountsDB::new(paths, operating_mode)),
account_locks: Mutex::new(HashSet::new()), account_locks: Mutex::new(HashSet::new()),
readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))), readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))),
}; };
@ -794,7 +796,6 @@ mod tests {
account::Account, account::Account,
epoch_schedule::EpochSchedule, epoch_schedule::EpochSchedule,
fee_calculator::FeeCalculator, fee_calculator::FeeCalculator,
genesis_config::OperatingMode,
hash::Hash, hash::Hash,
instruction::CompiledInstruction, instruction::CompiledInstruction,
message::Message, message::Message,
@ -818,7 +819,7 @@ mod tests {
) -> Vec<(Result<TransactionLoadResult>, Option<HashAgeKind>)> { ) -> Vec<(Result<TransactionLoadResult>, Option<HashAgeKind>)> {
let mut hash_queue = BlockhashQueue::new(100); let mut hash_queue = BlockhashQueue::new(100);
hash_queue.register_hash(&tx.message().recent_blockhash, &fee_calculator); hash_queue.register_hash(&tx.message().recent_blockhash, &fee_calculator);
let accounts = Accounts::new(Vec::new()); let accounts = Accounts::new(Vec::new(), OperatingMode::Development);
for ka in ka.iter() { for ka in ka.iter() {
accounts.store_slow(0, &ka.0, &ka.1); accounts.store_slow(0, &ka.0, &ka.1);
} }
@ -1384,7 +1385,7 @@ mod tests {
#[test] #[test]
fn test_load_by_program_slot() { fn test_load_by_program_slot() {
let accounts = Accounts::new(Vec::new()); let accounts = Accounts::new(Vec::new(), OperatingMode::Development);
// Load accounts owned by various programs into AccountsDB // Load accounts owned by various programs into AccountsDB
let pubkey0 = Pubkey::new_rand(); let pubkey0 = Pubkey::new_rand();
@ -1407,7 +1408,7 @@ mod tests {
#[test] #[test]
fn test_accounts_account_not_found() { fn test_accounts_account_not_found() {
let accounts = Accounts::new(Vec::new()); let accounts = Accounts::new(Vec::new(), OperatingMode::Development);
let mut error_counters = ErrorCounters::default(); let mut error_counters = ErrorCounters::default();
let ancestors = vec![(0, 0)].into_iter().collect(); let ancestors = vec![(0, 0)].into_iter().collect();
@ -1429,7 +1430,7 @@ mod tests {
#[test] #[test]
#[should_panic] #[should_panic]
fn test_accounts_empty_bank_hash() { fn test_accounts_empty_bank_hash() {
let accounts = Accounts::new(Vec::new()); let accounts = Accounts::new(Vec::new(), OperatingMode::Development);
accounts.bank_hash_at(1); accounts.bank_hash_at(1);
} }
@ -1445,7 +1446,7 @@ mod tests {
let account2 = Account::new(3, 0, &Pubkey::default()); let account2 = Account::new(3, 0, &Pubkey::default());
let account3 = Account::new(4, 0, &Pubkey::default()); let account3 = Account::new(4, 0, &Pubkey::default());
let accounts = Accounts::new(Vec::new()); let accounts = Accounts::new(Vec::new(), OperatingMode::Development);
accounts.store_slow(0, &keypair0.pubkey(), &account0); accounts.store_slow(0, &keypair0.pubkey(), &account0);
accounts.store_slow(0, &keypair1.pubkey(), &account1); accounts.store_slow(0, &keypair1.pubkey(), &account1);
accounts.store_slow(0, &keypair2.pubkey(), &account2); accounts.store_slow(0, &keypair2.pubkey(), &account2);
@ -1557,7 +1558,7 @@ mod tests {
let account1 = Account::new(2, 0, &Pubkey::default()); let account1 = Account::new(2, 0, &Pubkey::default());
let account2 = Account::new(3, 0, &Pubkey::default()); let account2 = Account::new(3, 0, &Pubkey::default());
let accounts = Accounts::new(Vec::new()); let accounts = Accounts::new(Vec::new(), OperatingMode::Development);
accounts.store_slow(0, &keypair0.pubkey(), &account0); accounts.store_slow(0, &keypair0.pubkey(), &account0);
accounts.store_slow(0, &keypair1.pubkey(), &account1); accounts.store_slow(0, &keypair1.pubkey(), &account1);
accounts.store_slow(0, &keypair2.pubkey(), &account2); accounts.store_slow(0, &keypair2.pubkey(), &account2);
@ -1687,7 +1688,7 @@ mod tests {
let mut loaded = vec![loaded0, loaded1]; let mut loaded = vec![loaded0, loaded1];
let accounts = Accounts::new(Vec::new()); let accounts = Accounts::new(Vec::new(), OperatingMode::Development);
{ {
let mut readonly_locks = accounts.readonly_locks.write().unwrap(); let mut readonly_locks = accounts.readonly_locks.write().unwrap();
let readonly_locks = readonly_locks.as_mut().unwrap(); let readonly_locks = readonly_locks.as_mut().unwrap();
@ -1738,7 +1739,7 @@ mod tests {
#[test] #[test]
fn huge_clean() { fn huge_clean() {
solana_logger::setup(); solana_logger::setup();
let accounts = Accounts::new(Vec::new()); let accounts = Accounts::new(Vec::new(), OperatingMode::Development);
let mut old_pubkey = Pubkey::default(); let mut old_pubkey = Pubkey::default();
let zero_account = Account::new(0, 0, &Account::default().owner); let zero_account = Account::new(0, 0, &Account::default().owner);
info!("storing.."); info!("storing..");

246
runtime/src/accounts_db.rs
View File

@ -22,7 +22,7 @@ use crate::{
accounts_index::{AccountsIndex, Ancestors, SlotList, SlotSlice}, accounts_index::{AccountsIndex, Ancestors, SlotList, SlotSlice},
append_vec::{AppendVec, StoredAccount, StoredMeta}, append_vec::{AppendVec, StoredAccount, StoredMeta},
}; };
use byteorder::{ByteOrder, LittleEndian}; use blake3::traits::digest::Digest;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use log::*; use log::*;
use rand::{thread_rng, Rng}; use rand::{thread_rng, Rng};
@ -33,9 +33,11 @@ use solana_rayon_threadlimit::get_thread_count;
use solana_sdk::{ use solana_sdk::{
account::Account, account::Account,
clock::{Epoch, Slot}, clock::{Epoch, Slot},
genesis_config::OperatingMode,
hash::{Hash, Hasher}, hash::{Hash, Hasher},
pubkey::Pubkey, pubkey::Pubkey,
}; };
use std::convert::TryFrom;
use std::{ use std::{
collections::{HashMap, HashSet}, collections::{HashMap, HashSet},
io::{Error as IOError, Result as IOResult}, io::{Error as IOError, Result as IOResult},
@ -413,6 +415,8 @@ pub struct AccountsDB {
dead_slots: RwLock<HashSet<Slot>>, dead_slots: RwLock<HashSet<Slot>>,
stats: AccountsStats, stats: AccountsStats,
operating_mode: Option<OperatingMode>,
} }
#[derive(Debug, Default)] #[derive(Debug, Default)]
@ -459,16 +463,18 @@ impl Default for AccountsDB {
frozen_accounts: HashMap::new(), frozen_accounts: HashMap::new(),
dead_slots: RwLock::new(HashSet::new()), dead_slots: RwLock::new(HashSet::new()),
stats: AccountsStats::default(), stats: AccountsStats::default(),
operating_mode: None,
} }
} }
} }
impl AccountsDB { impl AccountsDB {
pub fn new(paths: Vec<PathBuf>) -> Self { pub fn new(paths: Vec<PathBuf>, operating_mode: OperatingMode) -> Self {
let new = if !paths.is_empty() { let new = if !paths.is_empty() {
Self { Self {
paths, paths,
temp_paths: None, temp_paths: None,
operating_mode: Some(operating_mode),
..Self::default() ..Self::default()
} }
} else { } else {
@ -478,6 +484,7 @@ impl AccountsDB {
Self { Self {
paths, paths,
temp_paths: Some(temp_dirs), temp_paths: Some(temp_dirs),
operating_mode: Some(operating_mode),
..Self::default() ..Self::default()
} }
}; };
@ -493,14 +500,14 @@ impl AccountsDB {
pub fn new_single() -> Self { pub fn new_single() -> Self {
AccountsDB { AccountsDB {
min_num_stores: 0, min_num_stores: 0,
..AccountsDB::new(Vec::new()) ..AccountsDB::new(Vec::new(), OperatingMode::Development)
} }
} }
#[cfg(test)] #[cfg(test)]
pub fn new_sized(paths: Vec<PathBuf>, file_size: u64) -> Self { pub fn new_sized(paths: Vec<PathBuf>, file_size: u64) -> Self {
AccountsDB { AccountsDB {
file_size, file_size,
..AccountsDB::new(paths) ..AccountsDB::new(paths, OperatingMode::Development)
} }
} }
@ -1254,7 +1261,24 @@ impl AccountsDB {
assert!(self.storage.read().unwrap().0.get(&remove_slot).is_none()); assert!(self.storage.read().unwrap().0.get(&remove_slot).is_none());
} }
pub fn hash_stored_account(slot: Slot, account: &StoredAccount, include_owner: bool) -> Hash { pub fn hash_stored_account(
slot: Slot,
account: &StoredAccount,
operating_mode: OperatingMode,
include_owner: bool,
) -> Hash {
if slot > Self::get_blake3_slot(operating_mode) {
Self::blake3_hash_account_data(
slot,
account.account_meta.lamports,
&account.account_meta.owner,
account.account_meta.executable,
account.account_meta.rent_epoch,
account.data,
&account.meta.pubkey,
include_owner,
)
} else {
Self::hash_account_data( Self::hash_account_data(
slot, slot,
account.account_meta.lamports, account.account_meta.lamports,
@ -1266,13 +1290,27 @@ impl AccountsDB {
include_owner, include_owner,
) )
} }
}
pub fn hash_account( pub fn hash_account(
slot: Slot, slot: Slot,
account: &Account, account: &Account,
pubkey: &Pubkey, pubkey: &Pubkey,
operating_mode: OperatingMode,
include_owner: bool, include_owner: bool,
) -> Hash { ) -> Hash {
if slot > Self::get_blake3_slot(operating_mode) {
Self::blake3_hash_account_data(
slot,
account.lamports,
&account.owner,
account.executable,
account.rent_epoch,
&account.data,
pubkey,
include_owner,
)
} else {
Self::hash_account_data( Self::hash_account_data(
slot, slot,
account.lamports, account.lamports,
@ -1284,6 +1322,7 @@ impl AccountsDB {
include_owner, include_owner,
) )
} }
}
fn hash_frozen_account_data(account: &Account) -> Hash { fn hash_frozen_account_data(account: &Account) -> Hash {
let mut hasher = Hasher::default(); let mut hasher = Hasher::default();
@ -1322,16 +1361,12 @@ impl AccountsDB {
} }
let mut hasher = Hasher::default(); let mut hasher = Hasher::default();
let mut buf = [0u8; 8];
LittleEndian::write_u64(&mut buf[..], lamports); hasher.hash(&lamports.to_le_bytes());
hasher.hash(&buf);
LittleEndian::write_u64(&mut buf[..], slot); hasher.hash(&slot.to_le_bytes());
hasher.hash(&buf);
LittleEndian::write_u64(&mut buf[..], rent_epoch); hasher.hash(&rent_epoch.to_le_bytes());
hasher.hash(&buf);
hasher.hash(&data); hasher.hash(&data);
@ -1350,6 +1385,53 @@ impl AccountsDB {
hasher.result() hasher.result()
} }
pub fn blake3_hash_account_data(
slot: Slot,
lamports: u64,
owner: &Pubkey,
executable: bool,
rent_epoch: Epoch,
data: &[u8],
pubkey: &Pubkey,
include_owner: bool,
) -> Hash {
if lamports == 0 {
return Hash::default();
}
let mut hasher = blake3::Hasher::new();
hasher.update(&lamports.to_le_bytes());
hasher.update(&slot.to_le_bytes());
hasher.update(&rent_epoch.to_le_bytes());
hasher.update(&data);
if executable {
hasher.update(&[1u8; 1]);
} else {
hasher.update(&[0u8; 1]);
}
if include_owner {
hasher.update(&owner.as_ref());
}
hasher.update(&pubkey.as_ref());
Hash(<[u8; solana_sdk::hash::HASH_BYTES]>::try_from(hasher.finalize().as_slice()).unwrap())
}
fn get_blake3_slot(operating_mode: OperatingMode) -> Slot {
match operating_mode {
OperatingMode::Development => std::u64::MAX,
OperatingMode::Stable => std::u64::MAX,
OperatingMode::Preview => std::u64::MAX,
}
}
fn bulk_assign_write_version(&self, count: usize) -> u64 { fn bulk_assign_write_version(&self, count: usize) -> u64 {
self.write_version self.write_version
.fetch_add(count as u64, Ordering::Relaxed) .fetch_add(count as u64, Ordering::Relaxed)
@ -1572,6 +1654,8 @@ impl AccountsDB {
let hash = Self::hash_stored_account( let hash = Self::hash_stored_account(
*slot, *slot,
&account, &account,
self.operating_mode
.expect("Operating mode must be set at initialization"),
Self::include_owner_in_hash(*slot), Self::include_owner_in_hash(*slot),
); );
if hash != *account.hash { if hash != *account.hash {
@ -1808,13 +1892,24 @@ impl AccountsDB {
} }
} }
fn hash_accounts(&self, slot: Slot, accounts: &[(&Pubkey, &Account)]) -> Vec<Hash> { fn hash_accounts(
&self,
slot: Slot,
accounts: &[(&Pubkey, &Account)],
operating_mode: OperatingMode,
) -> Vec<Hash> {
let mut stats = BankHashStats::default(); let mut stats = BankHashStats::default();
let hashes: Vec<_> = accounts let hashes: Vec<_> = accounts
.iter() .iter()
.map(|(pubkey, account)| { .map(|(pubkey, account)| {
stats.update(account); stats.update(account);
Self::hash_account(slot, account, pubkey, Self::include_owner_in_hash(slot)) Self::hash_account(
slot,
account,
pubkey,
operating_mode,
Self::include_owner_in_hash(slot),
)
}) })
.collect(); .collect();
@ -1879,7 +1974,12 @@ impl AccountsDB {
/// Store the account update. /// Store the account update.
pub fn store(&self, slot: Slot, accounts: &[(&Pubkey, &Account)]) { pub fn store(&self, slot: Slot, accounts: &[(&Pubkey, &Account)]) {
self.assert_frozen_accounts(accounts); self.assert_frozen_accounts(accounts);
let hashes = self.hash_accounts(slot, accounts); let hashes = self.hash_accounts(
slot,
accounts,
self.operating_mode
.expect("Operating mode must be set at initialization"),
);
self.store_with_hashes(slot, accounts, &hashes); self.store_with_hashes(slot, accounts, &hashes);
} }
@ -2042,7 +2142,7 @@ pub mod tests {
#[test] #[test]
fn test_accountsdb_add_root() { fn test_accountsdb_add_root() {
solana_logger::setup(); solana_logger::setup();
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::default(); let key = Pubkey::default();
let account0 = Account::new(1, 0, &key); let account0 = Account::new(1, 0, &key);
@ -2055,7 +2155,7 @@ pub mod tests {
#[test] #[test]
fn test_accountsdb_latest_ancestor() { fn test_accountsdb_latest_ancestor() {
solana_logger::setup(); solana_logger::setup();
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::default(); let key = Pubkey::default();
let account0 = Account::new(1, 0, &key); let account0 = Account::new(1, 0, &key);
@ -2082,7 +2182,7 @@ pub mod tests {
#[test] #[test]
fn test_accountsdb_latest_ancestor_with_root() { fn test_accountsdb_latest_ancestor_with_root() {
solana_logger::setup(); solana_logger::setup();
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::default(); let key = Pubkey::default();
let account0 = Account::new(1, 0, &key); let account0 = Account::new(1, 0, &key);
@ -2102,7 +2202,7 @@ pub mod tests {
#[test] #[test]
fn test_accountsdb_root_one_slot() { fn test_accountsdb_root_one_slot() {
solana_logger::setup(); solana_logger::setup();
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::default(); let key = Pubkey::default();
let account0 = Account::new(1, 0, &key); let account0 = Account::new(1, 0, &key);
@ -2143,7 +2243,7 @@ pub mod tests {
#[test] #[test]
fn test_accountsdb_add_root_many() { fn test_accountsdb_add_root_many() {
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let mut pubkeys: Vec<Pubkey> = vec![]; let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&db, &mut pubkeys, 0, 100, 0, 0); create_account(&db, &mut pubkeys, 0, 100, 0, 0);
@ -2232,7 +2332,7 @@ pub mod tests {
let key = Pubkey::default(); let key = Pubkey::default();
// 1 token in the "root", i.e. db zero // 1 token in the "root", i.e. db zero
let db0 = AccountsDB::new(Vec::new()); let db0 = AccountsDB::new(Vec::new(), OperatingMode::Development);
let account0 = Account::new(1, 0, &key); let account0 = Account::new(1, 0, &key);
db0.store(0, &[(&key, &account0)]); db0.store(0, &[(&key, &account0)]);
@ -2251,7 +2351,7 @@ pub mod tests {
#[test] #[test]
fn test_remove_unrooted_slot() { fn test_remove_unrooted_slot() {
let unrooted_slot = 9; let unrooted_slot = 9;
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::default(); let key = Pubkey::default();
let account0 = Account::new(1, 0, &key); let account0 = Account::new(1, 0, &key);
let ancestors: HashMap<_, _> = vec![(unrooted_slot, 1)].into_iter().collect(); let ancestors: HashMap<_, _> = vec![(unrooted_slot, 1)].into_iter().collect();
@ -2297,7 +2397,7 @@ pub mod tests {
#[test] #[test]
fn test_remove_unrooted_slot_snapshot() { fn test_remove_unrooted_slot_snapshot() {
let unrooted_slot = 9; let unrooted_slot = 9;
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::new_rand(); let key = Pubkey::new_rand();
let account0 = Account::new(1, 0, &key); let account0 = Account::new(1, 0, &key);
db.store(unrooted_slot, &[(&key, &account0)]); db.store(unrooted_slot, &[(&key, &account0)]);
@ -2424,7 +2524,7 @@ pub mod tests {
#[test] #[test]
fn test_account_one() { fn test_account_one() {
let (_accounts_dirs, paths) = get_temp_accounts_paths(1).unwrap(); let (_accounts_dirs, paths) = get_temp_accounts_paths(1).unwrap();
let db = AccountsDB::new(paths); let db = AccountsDB::new(paths, OperatingMode::Development);
let mut pubkeys: Vec<Pubkey> = vec![]; let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&db, &mut pubkeys, 0, 1, 0, 0); create_account(&db, &mut pubkeys, 0, 1, 0, 0);
let ancestors = vec![(0, 0)].into_iter().collect(); let ancestors = vec![(0, 0)].into_iter().collect();
@ -2437,7 +2537,7 @@ pub mod tests {
#[test] #[test]
fn test_account_many() { fn test_account_many() {
let (_accounts_dirs, paths) = get_temp_accounts_paths(2).unwrap(); let (_accounts_dirs, paths) = get_temp_accounts_paths(2).unwrap();
let db = AccountsDB::new(paths); let db = AccountsDB::new(paths, OperatingMode::Development);
let mut pubkeys: Vec<Pubkey> = vec![]; let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&db, &mut pubkeys, 0, 100, 0, 0); create_account(&db, &mut pubkeys, 0, 100, 0, 0);
check_accounts(&db, &pubkeys, 0, 100, 1); check_accounts(&db, &pubkeys, 0, 100, 1);
@ -2555,7 +2655,7 @@ pub mod tests {
#[test] #[test]
fn test_purge_slot_not_root() { fn test_purge_slot_not_root() {
let accounts = AccountsDB::new(Vec::new()); let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development);
let mut pubkeys: Vec<Pubkey> = vec![]; let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0); create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
let ancestors = vec![(0, 0)].into_iter().collect(); let ancestors = vec![(0, 0)].into_iter().collect();
@ -2566,7 +2666,7 @@ pub mod tests {
#[test] #[test]
fn test_purge_slot_after_root() { fn test_purge_slot_after_root() {
let accounts = AccountsDB::new(Vec::new()); let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development);
let mut pubkeys: Vec<Pubkey> = vec![]; let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0); create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
let ancestors = vec![(0, 0)].into_iter().collect(); let ancestors = vec![(0, 0)].into_iter().collect();
@ -2581,7 +2681,7 @@ pub mod tests {
//This test is pedantic //This test is pedantic
//A slot is purged when a non root bank is cleaned up. If a slot is behind root but it is //A slot is purged when a non root bank is cleaned up. If a slot is behind root but it is
//not root, it means we are retaining dead banks. //not root, it means we are retaining dead banks.
let accounts = AccountsDB::new(Vec::new()); let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development);
let pubkey = Pubkey::new_rand(); let pubkey = Pubkey::new_rand();
let account = Account::new(1, 0, &Account::default().owner); let account = Account::new(1, 0, &Account::default().owner);
//store an account //store an account
@ -2659,7 +2759,7 @@ pub mod tests {
fn test_clean_old_with_normal_account() { fn test_clean_old_with_normal_account() {
solana_logger::setup(); solana_logger::setup();
let accounts = AccountsDB::new(Vec::new()); let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development);
let pubkey = Pubkey::new_rand(); let pubkey = Pubkey::new_rand();
let account = Account::new(1, 0, &Account::default().owner); let account = Account::new(1, 0, &Account::default().owner);
//store an account //store an account
@ -2685,7 +2785,7 @@ pub mod tests {
fn test_clean_old_with_zero_lamport_account() { fn test_clean_old_with_zero_lamport_account() {
solana_logger::setup(); solana_logger::setup();
let accounts = AccountsDB::new(Vec::new()); let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development);
let pubkey1 = Pubkey::new_rand(); let pubkey1 = Pubkey::new_rand();
let pubkey2 = Pubkey::new_rand(); let pubkey2 = Pubkey::new_rand();
let normal_account = Account::new(1, 0, &Account::default().owner); let normal_account = Account::new(1, 0, &Account::default().owner);
@ -2715,7 +2815,7 @@ pub mod tests {
fn test_clean_old_with_both_normal_and_zero_lamport_accounts() { fn test_clean_old_with_both_normal_and_zero_lamport_accounts() {
solana_logger::setup(); solana_logger::setup();
let accounts = AccountsDB::new(Vec::new()); let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development);
let pubkey1 = Pubkey::new_rand(); let pubkey1 = Pubkey::new_rand();
let pubkey2 = Pubkey::new_rand(); let pubkey2 = Pubkey::new_rand();
let normal_account = Account::new(1, 0, &Account::default().owner); let normal_account = Account::new(1, 0, &Account::default().owner);
@ -2748,7 +2848,7 @@ pub mod tests {
fn test_uncleaned_roots_with_account() { fn test_uncleaned_roots_with_account() {
solana_logger::setup(); solana_logger::setup();
let accounts = AccountsDB::new(Vec::new()); let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development);
let pubkey = Pubkey::new_rand(); let pubkey = Pubkey::new_rand();
let account = Account::new(1, 0, &Account::default().owner); let account = Account::new(1, 0, &Account::default().owner);
//store an account //store an account
@ -2768,7 +2868,7 @@ pub mod tests {
fn test_uncleaned_roots_with_no_account() { fn test_uncleaned_roots_with_no_account() {
solana_logger::setup(); solana_logger::setup();
let accounts = AccountsDB::new(Vec::new()); let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development);
assert_eq!(accounts.uncleaned_root_count(), 0); assert_eq!(accounts.uncleaned_root_count(), 0);
@ -3246,7 +3346,7 @@ pub mod tests {
#[test] #[test]
fn test_accountsdb_scan_accounts() { fn test_accountsdb_scan_accounts() {
solana_logger::setup(); solana_logger::setup();
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::default(); let key = Pubkey::default();
let key0 = Pubkey::new_rand(); let key0 = Pubkey::new_rand();
let account0 = Account::new(1, 0, &key); let account0 = Account::new(1, 0, &key);
@ -3310,7 +3410,7 @@ pub mod tests {
#[test] #[test]
fn test_store_large_account() { fn test_store_large_account() {
solana_logger::setup(); solana_logger::setup();
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::default(); let key = Pubkey::default();
let data_len = DEFAULT_FILE_SIZE as usize + 7; let data_len = DEFAULT_FILE_SIZE as usize + 7;
@ -3395,7 +3495,7 @@ pub mod tests {
fn test_frozen_account_lamport_increase() { fn test_frozen_account_lamport_increase() {
let frozen_pubkey = let frozen_pubkey =
Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap(); Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap();
let mut db = AccountsDB::new(Vec::new()); let mut db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let mut account = Account::new(1, 42, &frozen_pubkey); let mut account = Account::new(1, 42, &frozen_pubkey);
db.store(0, &[(&frozen_pubkey, &account)]); db.store(0, &[(&frozen_pubkey, &account)]);
@ -3430,7 +3530,7 @@ pub mod tests {
fn test_frozen_account_lamport_decrease() { fn test_frozen_account_lamport_decrease() {
let frozen_pubkey = let frozen_pubkey =
Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap(); Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap();
let mut db = AccountsDB::new(Vec::new()); let mut db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let mut account = Account::new(1, 42, &frozen_pubkey); let mut account = Account::new(1, 42, &frozen_pubkey);
db.store(0, &[(&frozen_pubkey, &account)]); db.store(0, &[(&frozen_pubkey, &account)]);
@ -3450,7 +3550,7 @@ pub mod tests {
fn test_frozen_account_nonexistent() { fn test_frozen_account_nonexistent() {
let frozen_pubkey = let frozen_pubkey =
Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap(); Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap();
let mut db = AccountsDB::new(Vec::new()); let mut db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let ancestors = vec![(0, 0)].into_iter().collect(); let ancestors = vec![(0, 0)].into_iter().collect();
db.freeze_accounts(&ancestors, &[frozen_pubkey]); db.freeze_accounts(&ancestors, &[frozen_pubkey]);
@ -3463,7 +3563,7 @@ pub mod tests {
fn test_frozen_account_data_modified() { fn test_frozen_account_data_modified() {
let frozen_pubkey = let frozen_pubkey =
Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap(); Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap();
let mut db = AccountsDB::new(Vec::new()); let mut db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let mut account = Account::new(1, 42, &frozen_pubkey); let mut account = Account::new(1, 42, &frozen_pubkey);
db.store(0, &[(&frozen_pubkey, &account)]); db.store(0, &[(&frozen_pubkey, &account)]);
@ -3512,17 +3612,28 @@ pub mod tests {
hash: &hash, hash: &hash,
}; };
let account = stored_account.clone_account(); let account = stored_account.clone_account();
let expected_account_hash_without_owner = let expected_account_hash =
Hash::from_str("GGTsxvxwnMsNfN6yYbBVQaRgvbVLfxeWnGXNyB8iXDyE").unwrap(); Hash::from_str("GGTsxvxwnMsNfN6yYbBVQaRgvbVLfxeWnGXNyB8iXDyE").unwrap();
assert_eq!( assert_eq!(
AccountsDB::hash_stored_account(slot, &stored_account, false), AccountsDB::hash_stored_account(
expected_account_hash_without_owner, slot,
&stored_account,
OperatingMode::Development,
false
),
expected_account_hash,
"StoredAccount's data layout might be changed; update hashing if needed." "StoredAccount's data layout might be changed; update hashing if needed."
); );
assert_eq!( assert_eq!(
AccountsDB::hash_account(slot, &account, &stored_account.meta.pubkey, false), AccountsDB::hash_account(
expected_account_hash_without_owner, slot,
&account,
&stored_account.meta.pubkey,
OperatingMode::Development,
false,
),
expected_account_hash,
"Account-based hashing must be consistent with StoredAccount-based one." "Account-based hashing must be consistent with StoredAccount-based one."
); );
@ -3530,12 +3641,23 @@ pub mod tests {
Hash::from_str("5iRNZVcAnq9JLYjSF2ibFhGEeq48r9Eq9HXxwm3BxywN").unwrap(); Hash::from_str("5iRNZVcAnq9JLYjSF2ibFhGEeq48r9Eq9HXxwm3BxywN").unwrap();
assert_eq!( assert_eq!(
AccountsDB::hash_stored_account(slot, &stored_account, true), AccountsDB::hash_stored_account(
slot,
&stored_account,
OperatingMode::Development,
true
),
expected_account_hash_with_owner, expected_account_hash_with_owner,
"StoredAccount's data layout might be changed; update hashing if needed (with owner)." "StoredAccount's data layout might be changed; update hashing if needed (with owner)."
); );
assert_eq!( assert_eq!(
AccountsDB::hash_account(slot, &account, &stored_account.meta.pubkey, true), AccountsDB::hash_account(
slot,
&account,
&stored_account.meta.pubkey,
OperatingMode::Development,
true
),
expected_account_hash_with_owner, expected_account_hash_with_owner,
"Account-based hashing must be consistent with StoredAccount-based one (with owner)." "Account-based hashing must be consistent with StoredAccount-based one (with owner)."
); );
@ -3544,7 +3666,7 @@ pub mod tests {
#[test] #[test]
fn test_bank_hash_stats() { fn test_bank_hash_stats() {
solana_logger::setup(); solana_logger::setup();
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::default(); let key = Pubkey::default();
let some_data_len = 5; let some_data_len = 5;
@ -3572,7 +3694,7 @@ pub mod tests {
fn test_verify_bank_hash() { fn test_verify_bank_hash() {
use BankHashVerificationError::*; use BankHashVerificationError::*;
solana_logger::setup(); solana_logger::setup();
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::default(); let key = Pubkey::default();
let some_data_len = 0; let some_data_len = 0;
@ -3610,7 +3732,7 @@ pub mod tests {
#[test] #[test]
fn test_verify_bank_hash_no_account() { fn test_verify_bank_hash_no_account() {
solana_logger::setup(); solana_logger::setup();
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let some_slot: Slot = 0; let some_slot: Slot = 0;
let ancestors = vec![(some_slot, 0)].into_iter().collect(); let ancestors = vec![(some_slot, 0)].into_iter().collect();
@ -3628,7 +3750,7 @@ pub mod tests {
fn test_verify_bank_hash_bad_account_hash() { fn test_verify_bank_hash_bad_account_hash() {
use BankHashVerificationError::*; use BankHashVerificationError::*;
solana_logger::setup(); solana_logger::setup();
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::default(); let key = Pubkey::default();
let some_data_len = 0; let some_data_len = 0;
@ -3638,7 +3760,7 @@ pub mod tests {
let accounts = &[(&key, &account)]; let accounts = &[(&key, &account)];
// update AccountsDB's bank hash but discard real account hashes // update AccountsDB's bank hash but discard real account hashes
db.hash_accounts(some_slot, accounts); db.hash_accounts(some_slot, accounts, OperatingMode::Development);
// provide bogus account hashes // provide bogus account hashes
let some_hash = Hash::new(&[0xca; HASH_BYTES]); let some_hash = Hash::new(&[0xca; HASH_BYTES]);
db.store_with_hashes(some_slot, accounts, &[some_hash]); db.store_with_hashes(some_slot, accounts, &[some_hash]);
@ -3652,7 +3774,7 @@ pub mod tests {
#[test] #[test]
fn test_bad_bank_hash() { fn test_bad_bank_hash() {
use solana_sdk::signature::{Keypair, Signer}; use solana_sdk::signature::{Keypair, Signer};
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let some_slot: Slot = 0; let some_slot: Slot = 0;
let ancestors: Ancestors = [(some_slot, 0)].iter().copied().collect(); let ancestors: Ancestors = [(some_slot, 0)].iter().copied().collect();
@ -3677,7 +3799,13 @@ pub mod tests {
for (key, account) in &accounts_keys { for (key, account) in &accounts_keys {
assert_eq!( assert_eq!(
db.load_account_hash(&ancestors, key), db.load_account_hash(&ancestors, key),
AccountsDB::hash_account(some_slot, &account, &key, false) AccountsDB::hash_account(
some_slot,
&account,
&key,
OperatingMode::Development,
false
)
); );
} }
} }
@ -3685,13 +3813,13 @@ pub mod tests {
#[test] #[test]
fn test_get_snapshot_storages_empty() { fn test_get_snapshot_storages_empty() {
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
assert!(db.get_snapshot_storages(0).is_empty()); assert!(db.get_snapshot_storages(0).is_empty());
} }
#[test] #[test]
fn test_get_snapshot_storages_only_older_than_or_equal_to_snapshot_slot() { fn test_get_snapshot_storages_only_older_than_or_equal_to_snapshot_slot() {
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::default(); let key = Pubkey::default();
let account = Account::new(1, 0, &key); let account = Account::new(1, 0, &key);
@ -3709,7 +3837,7 @@ pub mod tests {
#[test] #[test]
fn test_get_snapshot_storages_only_non_empty() { fn test_get_snapshot_storages_only_non_empty() {
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::default(); let key = Pubkey::default();
let account = Account::new(1, 0, &key); let account = Account::new(1, 0, &key);
@ -3733,7 +3861,7 @@ pub mod tests {
#[test] #[test]
fn test_get_snapshot_storages_only_roots() { fn test_get_snapshot_storages_only_roots() {
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::default(); let key = Pubkey::default();
let account = Account::new(1, 0, &key); let account = Account::new(1, 0, &key);
@ -3749,7 +3877,7 @@ pub mod tests {
#[test] #[test]
fn test_get_snapshot_storages_exclude_empty() { fn test_get_snapshot_storages_exclude_empty() {
let db = AccountsDB::new(Vec::new()); let db = AccountsDB::new(Vec::new(), OperatingMode::Development);
let key = Pubkey::default(); let key = Pubkey::default();
let account = Account::new(1, 0, &key); let account = Account::new(1, 0, &key);
@ -3768,7 +3896,7 @@ pub mod tests {
#[test] #[test]
#[should_panic(expected = "double remove of account in slot: 0/store: 0!!")] #[should_panic(expected = "double remove of account in slot: 0/store: 0!!")]
fn test_storage_remove_account_double_remove() { fn test_storage_remove_account_double_remove() {
let accounts = AccountsDB::new(Vec::new()); let accounts = AccountsDB::new(Vec::new(), OperatingMode::Development);
let pubkey = Pubkey::new_rand(); let pubkey = Pubkey::new_rand();
let account = Account::new(1, 0, &Account::default().owner); let account = Account::new(1, 0, &Account::default().owner);
accounts.store(0, &[(&pubkey, &account)]); accounts.store(0, &[(&pubkey, &account)]);

2
runtime/src/bank.rs
View File

@ -375,7 +375,7 @@ impl Bank {
bank.operating_mode = Some(genesis_config.operating_mode); bank.operating_mode = Some(genesis_config.operating_mode);
bank.ancestors.insert(bank.slot(), 0); bank.ancestors.insert(bank.slot(), 0);
bank.rc.accounts = Arc::new(Accounts::new(paths)); bank.rc.accounts = Arc::new(Accounts::new(paths, genesis_config.operating_mode));
bank.process_genesis_config(genesis_config); bank.process_genesis_config(genesis_config);
bank.finish_init(); bank.finish_init();

7
runtime/src/serde_snapshot.rs
View File

@ -15,7 +15,7 @@ use {
de::{DeserializeOwned, Visitor}, de::{DeserializeOwned, Visitor},
Deserialize, Deserializer, Serialize, Serializer, Deserialize, Deserializer, Serialize, Serializer,
}, },
solana_sdk::clock::Slot, solana_sdk::{clock::Slot, genesis_config::OperatingMode},
std::{ std::{
cmp::min, cmp::min,
collections::HashMap, collections::HashMap,
@ -104,6 +104,7 @@ pub fn bankrc_from_stream<R, P>(
slot: Slot, slot: Slot,
stream: &mut BufReader<R>, stream: &mut BufReader<R>,
stream_append_vecs_path: P, stream_append_vecs_path: P,
operating_mode: OperatingMode,
) -> std::result::Result<BankRc, IoError> ) -> std::result::Result<BankRc, IoError>
where where
R: Read, R: Read,
@ -116,6 +117,7 @@ where
$x::deserialize_accounts_db_fields(stream)?, $x::deserialize_accounts_db_fields(stream)?,
account_paths, account_paths,
stream_append_vecs_path, stream_append_vecs_path,
operating_mode,
)?), )?),
slot, slot,
)) ))
@ -190,12 +192,13 @@ fn context_accountsdb_from_fields<'a, C, P>(
account_db_fields: AccountDBFields<C::SerializableAccountStorageEntry>, account_db_fields: AccountDBFields<C::SerializableAccountStorageEntry>,
account_paths: &[PathBuf], account_paths: &[PathBuf],
stream_append_vecs_path: P, stream_append_vecs_path: P,
operating_mode: OperatingMode,
) -> Result<AccountsDB, IoError> ) -> Result<AccountsDB, IoError>
where where
C: TypeContext<'a>, C: TypeContext<'a>,
P: AsRef<Path>, P: AsRef<Path>,
{ {
let accounts_db = AccountsDB::new(account_paths.to_vec()); let accounts_db = AccountsDB::new(account_paths.to_vec(), operating_mode);
let AccountDBFields(storage, version, slot, bank_hash_info) = account_db_fields; let AccountDBFields(storage, version, slot, bank_hash_info) = account_db_fields;

6
runtime/src/serde_snapshot/tests.rs
View File

@ -10,7 +10,7 @@ use {
solana_sdk::{ solana_sdk::{
account::Account, account::Account,
clock::Slot, clock::Slot,
genesis_config::create_genesis_config, genesis_config::{create_genesis_config, OperatingMode},
pubkey::Pubkey, pubkey::Pubkey,
signature::{Keypair, Signer}, signature::{Keypair, Signer},
}, },
@ -68,6 +68,7 @@ where
C::deserialize_accounts_db_fields(stream)?, C::deserialize_accounts_db_fields(stream)?,
account_paths, account_paths,
stream_append_vecs_path, stream_append_vecs_path,
OperatingMode::Development,
) )
} }
@ -135,7 +136,7 @@ where
fn test_accounts_serialize_style(serde_style: SerdeStyle) { fn test_accounts_serialize_style(serde_style: SerdeStyle) {
solana_logger::setup(); solana_logger::setup();
let (_accounts_dir, paths) = get_temp_accounts_paths(4).unwrap(); let (_accounts_dir, paths) = get_temp_accounts_paths(4).unwrap();
let accounts = Accounts::new(paths); let accounts = Accounts::new(paths, OperatingMode::Development);
let mut pubkeys: Vec<Pubkey> = vec![]; let mut pubkeys: Vec<Pubkey> = vec![];
create_test_accounts(&accounts, &mut pubkeys, 100, 0); create_test_accounts(&accounts, &mut pubkeys, 100, 0);
@ -227,6 +228,7 @@ fn test_bank_serialize_style(serde_style: SerdeStyle) {
dbank.slot(), dbank.slot(),
&mut reader, &mut reader,
copied_accounts.path(), copied_accounts.path(),
OperatingMode::Development,
) )
.unwrap(), .unwrap(),
ref_sc, ref_sc,

2
sdk/src/hash.rs
View File

@ -8,7 +8,7 @@ use thiserror::Error;
pub const HASH_BYTES: usize = 32; pub const HASH_BYTES: usize = 32;
#[derive(Serialize, Deserialize, Clone, Copy, Default, Eq, PartialEq, Ord, PartialOrd, Hash)] #[derive(Serialize, Deserialize, Clone, Copy, Default, Eq, PartialEq, Ord, PartialOrd, Hash)]
#[repr(transparent)] #[repr(transparent)]
pub struct Hash([u8; HASH_BYTES]); pub struct Hash(pub [u8; HASH_BYTES]);
#[derive(Clone, Default)] #[derive(Clone, Default)]
pub struct Hasher { pub struct Hasher {