gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fail fast if account paths cannot be canonicalized (#7300) (#7315)

Browse Source 
automerge
This commit is contained in:
mergify[bot]
2019-12-05 19:45:39 -08:00
committed by Grimes
parent bae659b9c7
commit a1736606dc
13 changed files with 123 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
runtime/benches/accounts.rs
View File

@ -7,7 +7,7 @@ use solana_runtime::bank::*;
use solana_sdk::account::Account;
use solana_sdk::genesis_config::create_genesis_config;
use solana_sdk::pubkey::Pubkey;
use std::sync::Arc;
use std::{path::PathBuf, sync::Arc};
use test::Bencher;
fn deposit_many(bank: &Bank, pubkeys: &mut Vec<Pubkey>, num: usize) {
@ -24,7 +24,7 @@ fn deposit_many(bank: &Bank, pubkeys: &mut Vec<Pubkey>, num: usize) {
#[bench]
fn test_accounts_create(bencher: &mut Bencher) {
let (genesis_config, _) = create_genesis_config(10_000);
let bank0 = Bank::new_with_paths(&genesis_config, Some("bench_a0".to_string()));
let bank0 = Bank::new_with_paths(&genesis_config, vec![PathBuf::from("bench_a0")]);
bencher.iter(|| {
let mut pubkeys: Vec<Pubkey> = vec![];
deposit_many(&bank0, &mut pubkeys, 1000);
@ -37,7 +37,7 @@ fn test_accounts_squash(bencher: &mut Bencher) {
let mut banks: Vec<Arc<Bank>> = Vec::with_capacity(10);
banks.push(Arc::new(Bank::new_with_paths(
&genesis_config,
Some("bench_a1".to_string()),
vec![PathBuf::from("bench_a1")],
)));
let mut pubkeys: Vec<Pubkey> = vec![];
deposit_many(&banks[0], &mut pubkeys, 250000);
@ -60,7 +60,7 @@ fn test_accounts_squash(bencher: &mut Bencher) {
#[bench]
fn test_accounts_hash_internal_state(bencher: &mut Bencher) {
let accounts = Accounts::new(Some("bench_accounts_hash_internal".to_string()));
let accounts = Accounts::new(vec![PathBuf::from("bench_accounts_hash_internal")]);
let mut pubkeys: Vec<Pubkey> = vec![];
create_test_accounts(&accounts, &mut pubkeys, 60000, 0);
let ancestors = vec![(0, 0)].into_iter().collect();

28
runtime/src/accounts.rs
View File

@ -18,7 +18,7 @@ use solana_sdk::transaction::Result;
use solana_sdk::transaction::{Transaction, TransactionError};
use std::collections::{HashMap, HashSet};
use std::io::{BufReader, Error as IOError, Read};
use std::path::Path;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, RwLock};
use crate::transaction_utils::OrderedIterator;
@ -52,7 +52,7 @@ pub type TransactionLoaders = Vec<Vec<(Pubkey, Account)>>;
pub type TransactionLoadResult = (TransactionAccounts, TransactionLoaders, TransactionRent);
impl Accounts {
pub fn new(paths: Option<String>) -> Self {
pub fn new(paths: Vec<PathBuf>) -> Self {
let accounts_db = Arc::new(AccountsDB::new(paths));
Accounts {
@ -76,7 +76,7 @@ impl Accounts {
pub fn accounts_from_stream<R: Read, P: AsRef<Path>>(
&self,
stream: &mut BufReader<R>,
local_paths: String,
local_paths: &[PathBuf],
append_vecs_path: P,
) -> std::result::Result<(), IOError> {
self.accounts_db
@ -621,7 +621,7 @@ mod tests {
) -> Vec<Result<TransactionLoadResult>> {
let mut hash_queue = BlockhashQueue::new(100);
hash_queue.register_hash(&tx.message().recent_blockhash, &fee_calculator);
let accounts = Accounts::new(None);
let accounts = Accounts::new(Vec::new());
for ka in ka.iter() {
accounts.store_slow(0, &ka.0, &ka.1);
}
@ -1052,7 +1052,7 @@ mod tests {
#[test]
fn test_load_by_program_slot() {
let accounts = Accounts::new(None);
let accounts = Accounts::new(Vec::new());
// Load accounts owned by various programs into AccountsDB
let pubkey0 = Pubkey::new_rand();
@ -1075,7 +1075,7 @@ mod tests {
#[test]
fn test_accounts_account_not_found() {
let accounts = Accounts::new(None);
let accounts = Accounts::new(Vec::new());
let mut error_counters = ErrorCounters::default();
let ancestors = vec![(0, 0)].into_iter().collect();
@ -1097,14 +1097,14 @@ mod tests {
#[test]
#[should_panic]
fn test_accounts_empty_hash_internal_state() {
let accounts = Accounts::new(None);
let accounts = Accounts::new(Vec::new());
accounts.hash_internal_state(0);
}
#[test]
#[should_panic]
fn test_accounts_empty_account_hash_internal_state() {
let accounts = Accounts::new(None);
let accounts = Accounts::new(Vec::new());
accounts.store_slow(0, &Pubkey::default(), &Account::new(1, 0, &sysvar::id()));
accounts.hash_internal_state(0);
}
@ -1126,7 +1126,7 @@ mod tests {
fn test_accounts_serialize() {
solana_logger::setup();
let (_accounts_dir, paths) = get_temp_accounts_paths(4).unwrap();
let accounts = Accounts::new(Some(paths));
let accounts = Accounts::new(paths);
let mut pubkeys: Vec<Pubkey> = vec![];
create_test_accounts(&accounts, &mut pubkeys, 100, 0);
@ -1148,9 +1148,9 @@ mod tests {
let buf = writer.into_inner();
let mut reader = BufReader::new(&buf[..]);
let (_accounts_dir, daccounts_paths) = get_temp_accounts_paths(2).unwrap();
let daccounts = Accounts::new(Some(daccounts_paths.clone()));
let daccounts = Accounts::new(daccounts_paths.clone());
assert!(daccounts
.accounts_from_stream(&mut reader, daccounts_paths, copied_accounts.path())
.accounts_from_stream(&mut reader, &daccounts_paths, copied_accounts.path())
.is_ok());
check_accounts(&daccounts, &pubkeys, 100);
assert_eq!(
@ -1171,7 +1171,7 @@ mod tests {
let account2 = Account::new(3, 0, &Pubkey::default());
let account3 = Account::new(4, 0, &Pubkey::default());
let accounts = Accounts::new(None);
let accounts = Accounts::new(Vec::new());
accounts.store_slow(0, &keypair0.pubkey(), &account0);
accounts.store_slow(0, &keypair1.pubkey(), &account1);
accounts.store_slow(0, &keypair2.pubkey(), &account2);
@ -1283,7 +1283,7 @@ mod tests {
let account1 = Account::new(2, 0, &Pubkey::default());
let account2 = Account::new(3, 0, &Pubkey::default());
let accounts = Accounts::new(None);
let accounts = Accounts::new(Vec::new());
accounts.store_slow(0, &keypair0.pubkey(), &account0);
accounts.store_slow(0, &keypair1.pubkey(), &account1);
accounts.store_slow(0, &keypair2.pubkey(), &account2);
@ -1404,7 +1404,7 @@ mod tests {
let mut loaded = vec![loaded0, loaded1];
let accounts = Accounts::new(None);
let accounts = Accounts::new(Vec::new());
{
let mut readonly_locks = accounts.readonly_locks.write().unwrap();
let readonly_locks = readonly_locks.as_mut().unwrap();

79
runtime/src/accounts_db.rs
View File

@ -307,18 +307,11 @@ impl AccountStorageEntry {
}
}
pub fn get_paths_vec(paths: &str) -> Vec<PathBuf> {
paths.split(',').map(PathBuf::from).collect()
}
pub fn get_temp_accounts_paths(count: u32) -> IOResult<(Vec<TempDir>, String)> {
pub fn get_temp_accounts_paths(count: u32) -> IOResult<(Vec<TempDir>, Vec<PathBuf>)> {
let temp_dirs: IOResult<Vec<TempDir>> = (0..count).map(|_| TempDir::new()).collect();
let temp_dirs = temp_dirs?;
let paths: Vec<String> = temp_dirs
.iter()
.map(|t| t.path().to_str().unwrap().to_owned())
.collect();
Ok((temp_dirs, paths.join(",")))
let paths: Vec<PathBuf> = temp_dirs.iter().map(|t| t.path().to_path_buf()).collect();
Ok((temp_dirs, paths))
}
pub struct AccountsDBSerialize<'a> {
@ -413,10 +406,10 @@ impl Default for AccountsDB {
}
impl AccountsDB {
pub fn new(paths: Option<String>) -> Self {
if let Some(paths) = paths {
pub fn new(paths: Vec<PathBuf>) -> Self {
if !paths.is_empty() {
Self {
paths: RwLock::new(get_paths_vec(&paths)),
paths: RwLock::new(paths),
temp_paths: None,
..Self::default()
}
@ -425,7 +418,7 @@ impl AccountsDB {
// for testing
let (temp_dirs, paths) = get_temp_accounts_paths(DEFAULT_NUM_DIRS).unwrap();
Self {
paths: RwLock::new(get_paths_vec(&paths)),
paths: RwLock::new(paths),
temp_paths: Some(temp_dirs),
..Self::default()
}
@ -436,29 +429,21 @@ impl AccountsDB {
pub fn new_single() -> Self {
AccountsDB {
min_num_stores: 0,
..AccountsDB::new(None)
..AccountsDB::new(Vec::new())
}
}
#[cfg(test)]
pub fn new_sized(paths: Option<String>, file_size: u64) -> Self {
pub fn new_sized(paths: Vec<PathBuf>, file_size: u64) -> Self {
AccountsDB {
file_size,
..AccountsDB::new(paths)
}
}
pub fn format_paths<P: AsRef<Path>>(paths: Vec<P>) -> String {
let paths: Vec<String> = paths
.iter()
.map(|p| p.as_ref().to_str().unwrap().to_owned())
.collect();
paths.join(",")
}
pub fn accounts_from_stream<R: Read, P: AsRef<Path>>(
&self,
mut stream: &mut BufReader<R>,
local_account_paths: String,
local_account_paths: &[PathBuf],
append_vecs_path: P,
) -> Result<(), IOError> {
let _len: usize =
@ -467,7 +452,6 @@ impl AccountsDB {
deserialize_from(&mut stream).map_err(|e| AccountsDB::get_io_error(&e.to_string()))?;
// Remap the deserialized AppendVec paths to point to correct local paths
let local_account_paths = get_paths_vec(&local_account_paths);
let new_storage_map: Result<HashMap<Slot, SlotStores>, IOError> = storage
.0
.into_iter()
@ -534,7 +518,7 @@ impl AccountsDB {
.insert(slot_hash.0, slot_hash.1);
// Process deserialized data, set necessary fields in self
*self.paths.write().unwrap() = local_account_paths;
*self.paths.write().unwrap() = local_account_paths.to_vec();
let max_id: usize = *storage
.0
.values()
@ -1214,7 +1198,7 @@ pub mod tests {
#[test]
fn test_accountsdb_add_root() {
solana_logger::setup();
let db = AccountsDB::new(None);
let db = AccountsDB::new(Vec::new());
let key = Pubkey::default();
let account0 = Account::new(1, 0, &key);
@ -1227,7 +1211,7 @@ pub mod tests {
#[test]
fn test_accountsdb_latest_ancestor() {
solana_logger::setup();
let db = AccountsDB::new(None);
let db = AccountsDB::new(Vec::new());
let key = Pubkey::default();
let account0 = Account::new(1, 0, &key);
@ -1254,7 +1238,7 @@ pub mod tests {
#[test]
fn test_accountsdb_latest_ancestor_with_root() {
solana_logger::setup();
let db = AccountsDB::new(None);
let db = AccountsDB::new(Vec::new());
let key = Pubkey::default();
let account0 = Account::new(1, 0, &key);
@ -1274,7 +1258,7 @@ pub mod tests {
#[test]
fn test_accountsdb_root_one_slot() {
solana_logger::setup();
let db = AccountsDB::new(None);
let db = AccountsDB::new(Vec::new());
let key = Pubkey::default();
let account0 = Account::new(1, 0, &key);
@ -1315,7 +1299,7 @@ pub mod tests {
#[test]
fn test_accountsdb_add_root_many() {
let db = AccountsDB::new(None);
let db = AccountsDB::new(Vec::new());
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&db, &mut pubkeys, 0, 100, 0, 0);
@ -1383,7 +1367,7 @@ pub mod tests {
let key = Pubkey::default();
// 1 token in the "root", i.e. db zero
let db0 = AccountsDB::new(None);
let db0 = AccountsDB::new(Vec::new());
let account0 = Account::new(1, 0, &key);
db0.store(0, &[(&key, &account0)]);
@ -1492,7 +1476,7 @@ pub mod tests {
#[test]
fn test_account_one() {
let (_accounts_dirs, paths) = get_temp_accounts_paths(1).unwrap();
let db = AccountsDB::new(Some(paths));
let db = AccountsDB::new(paths);
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&db, &mut pubkeys, 0, 1, 0, 0);
let ancestors = vec![(0, 0)].into_iter().collect();
@ -1505,7 +1489,7 @@ pub mod tests {
#[test]
fn test_account_many() {
let (_accounts_dirs, paths) = get_temp_accounts_paths(2).unwrap();
let db = AccountsDB::new(Some(paths));
let db = AccountsDB::new(paths);
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&db, &mut pubkeys, 0, 100, 0, 0);
check_accounts(&db, &pubkeys, 0, 100, 1);
@ -1524,7 +1508,7 @@ pub mod tests {
fn test_account_grow_many() {
let (_accounts_dir, paths) = get_temp_accounts_paths(2).unwrap();
let size = 4096;
let accounts = AccountsDB::new_sized(Some(paths), size);
let accounts = AccountsDB::new_sized(paths, size);
let mut keys = vec![];
for i in 0..9 {
let key = Pubkey::new_rand();
@ -1623,7 +1607,7 @@ pub mod tests {
#[test]
fn test_purge_slot_not_root() {
let accounts = AccountsDB::new(None);
let accounts = AccountsDB::new(Vec::new());
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
let ancestors = vec![(0, 0)].into_iter().collect();
@ -1634,7 +1618,7 @@ pub mod tests {
#[test]
fn test_purge_slot_after_root() {
let accounts = AccountsDB::new(None);
let accounts = AccountsDB::new(Vec::new());
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
let ancestors = vec![(0, 0)].into_iter().collect();
@ -1648,7 +1632,7 @@ pub mod tests {
//This test is pedantic
//A slot is purged when a non root bank is cleaned up. If a slot is behind root but it is
//not root, it means we are retaining dead banks.
let accounts = AccountsDB::new(None);
let accounts = AccountsDB::new(Vec::new());
let pubkey = Pubkey::new_rand();
let account = Account::new(1, 0, &Account::default().owner);
//store an account
@ -1811,18 +1795,13 @@ pub mod tests {
let buf = writer.into_inner();
let mut reader = BufReader::new(&buf[..]);
let daccounts = AccountsDB::new(None);
let local_paths = {
let paths = daccounts.paths.read().unwrap();
AccountsDB::format_paths(paths.to_vec())
};
let daccounts = AccountsDB::new(Vec::new());
let local_paths = daccounts.paths.read().unwrap().clone();
let copied_accounts = TempDir::new().unwrap();
// Simulate obtaining a copy of the AppendVecs from a tarball
copy_append_vecs(&accounts, copied_accounts.path()).unwrap();
daccounts
.accounts_from_stream(&mut reader, local_paths, copied_accounts.path())
.accounts_from_stream(&mut reader, &local_paths, copied_accounts.path())
.unwrap();
print_count_and_status("daccounts", &daccounts);
@ -2023,7 +2002,7 @@ pub mod tests {
let min_file_bytes = std::mem::size_of::<StoredMeta>()
+ std::mem::size_of::<crate::append_vec::AccountMeta>();
let db = Arc::new(AccountsDB::new_sized(None, min_file_bytes as u64));
let db = Arc::new(AccountsDB::new_sized(Vec::new(), min_file_bytes as u64));
db.add_root(slot_id);
let thread_hdls: Vec<_> = (0..num_threads)
@ -2060,7 +2039,7 @@ pub mod tests {
#[test]
fn test_accountsdb_scan_accounts() {
solana_logger::setup();
let db = AccountsDB::new(None);
let db = AccountsDB::new(Vec::new());
let key = Pubkey::default();
let key0 = Pubkey::new_rand();
let account0 = Account::new(1, 0, &key);
@ -2093,7 +2072,7 @@ pub mod tests {
#[test]
fn test_store_large_account() {
solana_logger::setup();
let db = AccountsDB::new(None);
let db = AccountsDB::new(Vec::new());
let key = Pubkey::default();
let data_len = DEFAULT_FILE_SIZE as usize + 7;

16
runtime/src/bank.rs
View File

@ -47,7 +47,7 @@ use solana_sdk::{
use std::{
collections::HashMap,
io::{BufReader, Cursor, Error as IOError, Read},
path::Path,
path::{Path, PathBuf},
sync::atomic::{AtomicBool, AtomicU64, Ordering},
sync::{Arc, RwLock, RwLockReadGuard},
};
@ -69,8 +69,8 @@ pub struct BankRc {
}
impl BankRc {
pub fn new(account_paths: String, id: AppendVecId, slot: Slot) -> Self {
let accounts = Accounts::new(Some(account_paths));
pub fn new(account_paths: Vec<PathBuf>, id: AppendVecId, slot: Slot) -> Self {
let accounts = Accounts::new(account_paths);
accounts
.accounts_db
.next_id
@ -85,7 +85,7 @@ impl BankRc {
pub fn accounts_from_stream<R: Read, P: AsRef<Path>>(
&self,
mut stream: &mut BufReader<R>,
local_paths: String,
local_paths: &[PathBuf],
append_vecs_path: P,
) -> std::result::Result<(), IOError> {
let _len: usize =
@ -288,10 +288,10 @@ impl Default for BlockhashQueue {
impl Bank {
pub fn new(genesis_config: &GenesisConfig) -> Self {
Self::new_with_paths(&genesis_config, None)
Self::new_with_paths(&genesis_config, Vec::new())
}
pub fn new_with_paths(genesis_config: &GenesisConfig, paths: Option<String>) -> Self {
pub fn new_with_paths(genesis_config: &GenesisConfig, paths: Vec<PathBuf>) -> Self {
let mut bank = Self::default();
bank.ancestors.insert(bank.slot(), 0);
bank.rc.accounts = Arc::new(Accounts::new(paths));
@ -416,7 +416,7 @@ impl Bank {
pub fn create_with_genesis(
genesis_config: &GenesisConfig,
account_paths: String,
account_paths: Vec<PathBuf>,
status_cache_rc: &StatusCacheRc,
id: AppendVecId,
) -> Self {
@ -3678,7 +3678,7 @@ mod tests {
copy_append_vecs(&bank2.rc.accounts.accounts_db, copied_accounts.path()).unwrap();
dbank
.rc
.accounts_from_stream(&mut reader, dbank_paths, copied_accounts.path())
.accounts_from_stream(&mut reader, &dbank_paths, copied_accounts.path())
.unwrap();
assert_eq!(dbank.get_balance(&key1.pubkey()), 0);
assert_eq!(dbank.get_balance(&key2.pubkey()), 10);