From 634f4eb37d48f3d258e6c9c9c995f281dfef6b65 Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang <93241502+yhchiang-sol@users.noreply.github.com> Date: Wed, 2 Mar 2022 18:30:22 -0800 Subject: [PATCH] (LedgerStore) Use different path for different blockstore storage type. (#23236) #### Summary of Changes To avoid mixing the use of different shred storage types, each shred storage type will have its blockstore in a different directory. This PR still keeps the RocksFifo setting hidden. The default ShredStorageType and blockstore directory are still RocksLevel and `rocksdb`. Will follow-up with PRs on making FIFO option public in ledger-tool and validator. #### Test Plan * Added a new test to verify the existence of `rocksdb-fifo` directory when FIFO compaction is used. * Updated existing test to verify the current setting still store ledger under `rocksdb` directory. * Manually ran ledger_cleanup_test with both level and fifo compaction and verified the resulting ledger. * Ran a validator with this PR. --- genesis/src/main.rs | 6 +- ledger-tool/src/main.rs | 5 +- ledger/src/blockstore.rs | 114 +++++++++++++++++++++++++++++---- ledger/src/blockstore_db.rs | 8 +++ runtime/src/hardened_unpack.rs | 39 +++++++++++ test-validator/src/lib.rs | 5 +- 6 files changed, 161 insertions(+), 16 deletions(-) diff --git a/genesis/src/main.rs b/genesis/src/main.rs index 61e9c513c2..afd56f109b 100644 --- a/genesis/src/main.rs +++ b/genesis/src/main.rs @@ -13,7 +13,10 @@ use { }, solana_entry::poh::compute_hashes_per_tick, solana_genesis::{genesis_accounts::add_genesis_accounts, Base64Account}, - solana_ledger::{blockstore::create_new_ledger, blockstore_db::AccessType}, + solana_ledger::{ + blockstore::create_new_ledger, + blockstore_db::{AccessType, ShredStorageType}, + }, solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, solana_sdk::{ account::{Account, AccountSharedData, ReadableAccount, WritableAccount}, @@ -630,6 +633,7 @@ fn main() -> Result<(), Box> { &genesis_config, max_genesis_archive_unpacked_size, AccessType::PrimaryOnly, + ShredStorageType::default(), )?; println!("{}", genesis_config); diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index a7a822a829..417602e07f 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -23,7 +23,9 @@ use { ancestor_iterator::AncestorIterator, bank_forks_utils, blockstore::{create_new_ledger, Blockstore, PurgeType}, - blockstore_db::{self, AccessType, BlockstoreOptions, BlockstoreRecoveryMode, Database}, + blockstore_db::{ + self, AccessType, BlockstoreOptions, BlockstoreRecoveryMode, Database, ShredStorageType, + }, blockstore_processor::ProcessOptions, shred::Shred, }, @@ -1715,6 +1717,7 @@ fn main() { &genesis_config, solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, AccessType::PrimaryOnly, + ShredStorageType::default(), ) .unwrap_or_else(|err| { eprintln!("Failed to write genesis config: {:?}", err); diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 5cdc4f629f..e8df781d1f 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -6,7 +6,7 @@ use { ancestor_iterator::AncestorIterator, blockstore_db::{ columns as cf, AccessType, BlockstoreOptions, Column, Database, IteratorDirection, - IteratorMode, LedgerColumn, Result, WriteBatch, + IteratorMode, LedgerColumn, Result, ShredStorageType, WriteBatch, }, blockstore_meta::*, leader_schedule_cache::LeaderScheduleCache, @@ -72,7 +72,8 @@ pub use { pub mod blockstore_purge; -pub const BLOCKSTORE_DIRECTORY: &str = "rocksdb"; +pub const BLOCKSTORE_DIRECTORY_ROCKS_LEVEL: &str = "rocksdb"; +pub const BLOCKSTORE_DIRECTORY_ROCKS_FIFO: &str = "rocksdb_fifo"; thread_local!(static PAR_THREAD_POOL: RefCell = RefCell::new(rayon::ThreadPoolBuilder::new() .num_threads(get_thread_count()) @@ -354,10 +355,19 @@ impl Blockstore { self.db } + /// The path to the ledger store pub fn ledger_path(&self) -> &PathBuf { &self.ledger_path } + /// The directory under `ledger_path` to the underlying blockstore. + pub fn blockstore_directory(shred_storage_type: &ShredStorageType) -> &str { + match shred_storage_type { + ShredStorageType::RocksLevel => BLOCKSTORE_DIRECTORY_ROCKS_LEVEL, + ShredStorageType::RocksFifo(_) => BLOCKSTORE_DIRECTORY_ROCKS_FIFO, + } + } + /// Opens a Ledger in directory, provides "infinite" window of shreds pub fn open(ledger_path: &Path) -> Result { Self::do_open(ledger_path, BlockstoreOptions::default()) @@ -369,7 +379,8 @@ impl Blockstore { fn do_open(ledger_path: &Path, options: BlockstoreOptions) -> Result { fs::create_dir_all(&ledger_path)?; - let blockstore_path = ledger_path.join(BLOCKSTORE_DIRECTORY); + let blockstore_path = + ledger_path.join(Self::blockstore_directory(&options.shred_storage_type)); adjust_ulimit_nofile(options.enforce_ulimit_nofile)?; @@ -547,11 +558,15 @@ impl Blockstore { } /// Deletes the blockstore at the specified path. + /// + /// Note that if the `ledger_path` has multiple rocksdb instances, this + /// function will destroy all. pub fn destroy(ledger_path: &Path) -> Result<()> { - // Database::destroy() fails if the path doesn't exist + // Database::destroy() fails if the root directory doesn't exist fs::create_dir_all(ledger_path)?; - let blockstore_path = ledger_path.join(BLOCKSTORE_DIRECTORY); - Database::destroy(&blockstore_path) + Database::destroy(&Path::new(ledger_path).join(BLOCKSTORE_DIRECTORY_ROCKS_LEVEL)).and( + Database::destroy(&Path::new(ledger_path).join(BLOCKSTORE_DIRECTORY_ROCKS_FIFO)), + ) } /// Returns the SlotMeta of the specified slot. @@ -3783,18 +3798,20 @@ pub fn create_new_ledger( genesis_config: &GenesisConfig, max_genesis_archive_unpacked_size: u64, access_type: AccessType, + shred_storage_type: ShredStorageType, ) -> Result { Blockstore::destroy(ledger_path)?; genesis_config.write(ledger_path)?; // Fill slot 0 with ticks that link back to the genesis_config to bootstrap the ledger. + let blockstore_dir = Blockstore::blockstore_directory(&shred_storage_type); let blockstore = Blockstore::open_with_options( ledger_path, BlockstoreOptions { access_type, recovery_mode: None, enforce_ulimit_nofile: false, - ..BlockstoreOptions::default() + shred_storage_type: shred_storage_type.clone(), }, )?; let ticks_per_slot = genesis_config.ticks_per_slot; @@ -3827,7 +3844,7 @@ pub fn create_new_ledger( "-C", ledger_path.to_str().unwrap(), DEFAULT_GENESIS_FILE, - "rocksdb", + blockstore_dir, ]; let output = std::process::Command::new("tar") .args(&args) @@ -3884,11 +3901,11 @@ pub fn create_new_ledger( ) }); fs::rename( - &ledger_path.join("rocksdb"), - ledger_path.join("rocksdb.failed"), + &ledger_path.join(blockstore_dir), + ledger_path.join(format!("{}.failed", blockstore_dir)), ) .unwrap_or_else(|e| { - error_messages += &format!("/failed to stash problematic rocksdb: {}", e) + error_messages += &format!("/failed to stash problematic {}: {}", blockstore_dir, e) }); return Err(BlockstoreError::Io(IoError::new( @@ -3964,6 +3981,7 @@ macro_rules! create_new_tmp_ledger { $crate::tmp_ledger_name!(), $genesis_config, $crate::blockstore_db::AccessType::PrimaryOnly, + $crate::blockstore_db::ShredStorageType::default(), ) }; } @@ -3975,6 +3993,21 @@ macro_rules! create_new_tmp_ledger_auto_delete { $crate::tmp_ledger_name!(), $genesis_config, $crate::blockstore_db::AccessType::PrimaryOnly, + $crate::blockstore_db::ShredStorageType::default(), + ) + }; +} + +#[macro_export] +macro_rules! create_new_tmp_ledger_fifo_auto_delete { + ($genesis_config:expr) => { + $crate::blockstore::create_new_ledger_from_name_auto_delete( + $crate::tmp_ledger_name!(), + $genesis_config, + $crate::blockstore_db::AccessType::PrimaryOnly, + $crate::blockstore_db::ShredStorageType::RocksFifo( + $crate::blockstore_db::BlockstoreRocksFifoOptions::default(), + ), ) }; } @@ -4005,9 +4038,14 @@ pub fn create_new_ledger_from_name( name: &str, genesis_config: &GenesisConfig, access_type: AccessType, + shred_storage_type: ShredStorageType, ) -> (PathBuf, Hash) { - let (ledger_path, blockhash) = - create_new_ledger_from_name_auto_delete(name, genesis_config, access_type); + let (ledger_path, blockhash) = create_new_ledger_from_name_auto_delete( + name, + genesis_config, + access_type, + shred_storage_type, + ); (ledger_path.into_path(), blockhash) } @@ -4019,6 +4057,7 @@ pub fn create_new_ledger_from_name_auto_delete( name: &str, genesis_config: &GenesisConfig, access_type: AccessType, + shred_storage_type: ShredStorageType, ) -> (TempDir, Hash) { let ledger_path = get_ledger_path_from_name_auto_delete(name); let blockhash = create_new_ledger( @@ -4026,6 +4065,7 @@ pub fn create_new_ledger_from_name_auto_delete( genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, access_type, + shred_storage_type, ) .unwrap(); (ledger_path, blockhash) @@ -4168,6 +4208,7 @@ pub mod tests { use { super::*, crate::{ + blockstore_db::BlockstoreRocksFifoOptions, genesis_utils::{create_genesis_config, GenesisConfigInfo}, leader_schedule::{FixedSchedule, LeaderSchedule}, shred::{max_ticks_per_n_shreds, DataShredHeader}, @@ -4224,6 +4265,53 @@ pub mod tests { let entries = blockstore.get_slot_entries(0, 0).unwrap(); assert_eq!(ticks, entries); + assert!(Path::new(ledger_path.path()) + .join(Blockstore::blockstore_directory( + &ShredStorageType::RocksLevel, + )) + .exists()); + } + + #[test] + fn test_create_new_ledger_with_options_fifo() { + solana_logger::setup(); + let mint_total = 1_000_000_000_000; + let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(mint_total); + let (ledger_path, _blockhash) = create_new_tmp_ledger_fifo_auto_delete!(&genesis_config); + let blockstore = Blockstore::open_with_options( + ledger_path.path(), + BlockstoreOptions { + shred_storage_type: ShredStorageType::RocksFifo( + BlockstoreRocksFifoOptions::default(), + ), + ..BlockstoreOptions::default() + }, + ) + .unwrap(); + + let ticks = create_ticks(genesis_config.ticks_per_slot, 0, genesis_config.hash()); + let entries = blockstore.get_slot_entries(0, 0).unwrap(); + + assert_eq!(ticks, entries); + assert!(Path::new(ledger_path.path()) + .join(Blockstore::blockstore_directory( + &ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions::default()) + )) + .exists()); + } + + #[test] + fn test_rocksdb_directory() { + assert_eq!( + Blockstore::blockstore_directory(&ShredStorageType::RocksLevel), + BLOCKSTORE_DIRECTORY_ROCKS_LEVEL + ); + assert_eq!( + Blockstore::blockstore_directory(&ShredStorageType::RocksFifo( + BlockstoreRocksFifoOptions::default() + )), + BLOCKSTORE_DIRECTORY_ROCKS_FIFO + ); } #[test] diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index 44f721a515..4c111ea38f 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -971,6 +971,7 @@ pub struct WriteBatch<'a> { map: HashMap<&'static str, &'a ColumnFamily>, } +#[derive(Clone)] pub enum ShredStorageType { // Stores shreds under RocksDB's default compaction (level). RocksLevel, @@ -980,6 +981,12 @@ pub enum ShredStorageType { RocksFifo(BlockstoreRocksFifoOptions), } +impl Default for ShredStorageType { + fn default() -> Self { + Self::RocksLevel + } +} + pub struct BlockstoreOptions { // The access type of blockstore. Default: PrimaryOnly pub access_type: AccessType, @@ -1003,6 +1010,7 @@ impl Default for BlockstoreOptions { } } +#[derive(Clone)] pub struct BlockstoreRocksFifoOptions { // The maximum storage size for storing data shreds in column family // [`cf::DataShred`]. Typically, data shreds contribute around 25% of the diff --git a/runtime/src/hardened_unpack.rs b/runtime/src/hardened_unpack.rs index a889085970..87e727cd7f 100644 --- a/runtime/src/hardened_unpack.rs +++ b/runtime/src/hardened_unpack.rs @@ -467,6 +467,9 @@ fn is_valid_genesis_archive_entry(parts: &[&str], kind: tar::EntryType) -> bool (["rocksdb"], Directory) => true, (["rocksdb", _], GNUSparse) => true, (["rocksdb", _], Regular) => true, + (["rocksdb_fifo"], Directory) => true, + (["rocksdb_fifo", _], GNUSparse) => true, + (["rocksdb_fifo", _], Regular) => true, _ => false, } } @@ -600,6 +603,18 @@ mod tests { &["rocksdb", "foo"], tar::EntryType::GNUSparse, )); + assert!(is_valid_genesis_archive_entry( + &["rocksdb_fifo"], + tar::EntryType::Directory + )); + assert!(is_valid_genesis_archive_entry( + &["rocksdb_fifo", "foo"], + tar::EntryType::Regular + )); + assert!(is_valid_genesis_archive_entry( + &["rocksdb_fifo", "foo"], + tar::EntryType::GNUSparse, + )); assert!(!is_valid_genesis_archive_entry( &["aaaa"], @@ -633,6 +648,30 @@ mod tests { &["rocksdb", "foo", "bar"], tar::EntryType::GNUSparse )); + assert!(!is_valid_genesis_archive_entry( + &["rocksdb_fifo"], + tar::EntryType::Regular + )); + assert!(!is_valid_genesis_archive_entry( + &["rocksdb_fifo"], + tar::EntryType::GNUSparse, + )); + assert!(!is_valid_genesis_archive_entry( + &["rocksdb_fifo", "foo"], + tar::EntryType::Directory, + )); + assert!(!is_valid_genesis_archive_entry( + &["rocksdb_fifo", "foo", "bar"], + tar::EntryType::Directory, + )); + assert!(!is_valid_genesis_archive_entry( + &["rocksdb_fifo", "foo", "bar"], + tar::EntryType::Regular + )); + assert!(!is_valid_genesis_archive_entry( + &["rocksdb_fifo", "foo", "bar"], + tar::EntryType::GNUSparse + )); } fn with_finalize_and_unpack(archive: tar::Builder>, checker: C) -> Result<()> diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index 6e3bee2b5e..87d9b94703 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -12,7 +12,9 @@ use { gossip_service::discover_cluster, socketaddr, }, - solana_ledger::{blockstore::create_new_ledger, create_new_tmp_ledger}, + solana_ledger::{ + blockstore::create_new_ledger, blockstore_db::ShredStorageType, create_new_tmp_ledger, + }, solana_net_utils::PortRange, solana_rpc::{rpc::JsonRpcConfig, rpc_pubsub_service::PubSubConfig}, solana_runtime::{ @@ -580,6 +582,7 @@ impl TestValidator { .max_genesis_archive_unpacked_size .unwrap_or(MAX_GENESIS_ARCHIVE_UNPACKED_SIZE), solana_ledger::blockstore_db::AccessType::PrimaryOnly, + ShredStorageType::default(), ) .map_err(|err| { format!(