diff --git a/core/src/validator.rs b/core/src/validator.rs index aaf4adbcd7..cfe7436a14 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -36,7 +36,7 @@ use { solana_ledger::{ bank_forks_utils, blockstore::{Blockstore, BlockstoreSignals, CompletedSlotsReceiver, PurgeType}, - blockstore_db::{BlockstoreOptions, BlockstoreRecoveryMode, ShredStorageType}, + blockstore_db::{BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRecoveryMode}, blockstore_processor::{self, TransactionStatusSender}, leader_schedule::FixedSchedule, leader_schedule_cache::LeaderScheduleCache, @@ -165,7 +165,7 @@ pub struct ValidatorConfig { pub no_wait_for_vote_to_start_leader: bool, pub accounts_shrink_ratio: AccountShrinkThreshold, pub wait_to_vote_slot: Option, - pub shred_storage_type: ShredStorageType, + pub blockstore_advanced_options: BlockstoreAdvancedOptions, } impl Default for ValidatorConfig { @@ -226,7 +226,7 @@ impl Default for ValidatorConfig { accounts_shrink_ratio: AccountShrinkThreshold::default(), accounts_db_config: None, wait_to_vote_slot: None, - shred_storage_type: ShredStorageType::RocksLevel, + blockstore_advanced_options: BlockstoreAdvancedOptions::default(), } } } @@ -1259,7 +1259,7 @@ fn new_banks_from_ledger( BlockstoreOptions { recovery_mode: config.wal_recovery_mode.clone(), enforce_ulimit_nofile, - shred_storage_type: config.shred_storage_type.clone(), + advanced_options: config.blockstore_advanced_options.clone(), ..BlockstoreOptions::default() }, ) diff --git a/core/tests/ledger_cleanup.rs b/core/tests/ledger_cleanup.rs index f9fbef8b79..436e58a36f 100644 --- a/core/tests/ledger_cleanup.rs +++ b/core/tests/ledger_cleanup.rs @@ -9,7 +9,10 @@ mod tests { solana_core::ledger_cleanup_service::LedgerCleanupService, solana_ledger::{ blockstore::{make_many_slot_shreds, Blockstore}, - blockstore_db::{BlockstoreOptions, BlockstoreRocksFifoOptions, ShredStorageType}, + blockstore_db::{ + BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRocksFifoOptions, + ShredStorageType, + }, get_tmp_ledger_path, }, solana_measure::measure::Measure, @@ -348,10 +351,14 @@ mod tests { &ledger_path, if config.fifo_compaction { BlockstoreOptions { - shred_storage_type: ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions { - shred_data_cf_size: config.shred_data_cf_size, - ..BlockstoreRocksFifoOptions::default() - }), + advanced_options: BlockstoreAdvancedOptions { + shred_storage_type: ShredStorageType::RocksFifo( + BlockstoreRocksFifoOptions { + shred_data_cf_size: config.shred_data_cf_size, + ..BlockstoreRocksFifoOptions::default() + }, + ), + }, ..BlockstoreOptions::default() } } else { diff --git a/genesis/src/main.rs b/genesis/src/main.rs index afd56f109b..9ccdaa8d04 100644 --- a/genesis/src/main.rs +++ b/genesis/src/main.rs @@ -15,7 +15,7 @@ use { solana_genesis::{genesis_accounts::add_genesis_accounts, Base64Account}, solana_ledger::{ blockstore::create_new_ledger, - blockstore_db::{AccessType, ShredStorageType}, + blockstore_db::{AccessType, BlockstoreAdvancedOptions}, }, solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, solana_sdk::{ @@ -633,7 +633,7 @@ fn main() -> Result<(), Box> { &genesis_config, max_genesis_archive_unpacked_size, AccessType::PrimaryOnly, - ShredStorageType::default(), + BlockstoreAdvancedOptions::default(), )?; println!("{}", genesis_config); diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 5c3e98e8b7..6467112d60 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -24,7 +24,8 @@ use { bank_forks_utils, blockstore::{create_new_ledger, Blockstore, PurgeType}, blockstore_db::{ - self, AccessType, BlockstoreOptions, BlockstoreRecoveryMode, Database, ShredStorageType, + self, AccessType, BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRecoveryMode, + Database, }, blockstore_processor::ProcessOptions, shred::Shred, @@ -1721,7 +1722,7 @@ fn main() { &genesis_config, solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, AccessType::PrimaryOnly, - ShredStorageType::default(), + BlockstoreAdvancedOptions::default(), ) .unwrap_or_else(|err| { eprintln!("Failed to write genesis config: {:?}", err); diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index cb52a6c643..0aac089cb5 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -5,8 +5,9 @@ use { crate::{ ancestor_iterator::AncestorIterator, blockstore_db::{ - columns as cf, AccessType, BlockstoreOptions, Column, ColumnName, Database, - IteratorDirection, IteratorMode, LedgerColumn, Result, ShredStorageType, WriteBatch, + columns as cf, AccessType, BlockstoreAdvancedOptions, BlockstoreOptions, Column, + ColumnName, Database, IteratorDirection, IteratorMode, LedgerColumn, Result, + ShredStorageType, WriteBatch, }, blockstore_meta::*, leader_schedule_cache::LeaderScheduleCache, @@ -177,6 +178,7 @@ pub struct Blockstore { pub lowest_cleanup_slot: Arc>, no_compaction: bool, slots_stats: Arc>, + advanced_options: BlockstoreAdvancedOptions, } struct SlotsStats { @@ -519,8 +521,20 @@ impl BlockstoreRocksDbColumnFamilyMetrics { } macro_rules! rocksdb_metric_header { - ($metric_name:literal, $cf_name:literal) => { - concat!($metric_name, ",cf_name=", $cf_name) + ($metric_name:literal, $cf_name:literal, $advanced_options:expr) => { + match $advanced_options.shred_storage_type { + ShredStorageType::RocksLevel => + rocksdb_metric_header!(@all_fields $metric_name, $cf_name, "rocks_level"), + ShredStorageType::RocksFifo(_) => + rocksdb_metric_header!(@all_fields $metric_name, $cf_name, "rocks_fifo"), + } + }; + + (@all_fields $metric_name:literal, $cf_name:literal, $storage_type:literal) => { + concat!($metric_name, + ",cf_name=", $cf_name, + ",storage=", $storage_type, + ) }; } use rocksdb_metric_header; @@ -554,8 +568,10 @@ impl Blockstore { fn do_open(ledger_path: &Path, options: BlockstoreOptions) -> Result { fs::create_dir_all(&ledger_path)?; - let blockstore_path = - ledger_path.join(Self::blockstore_directory(&options.shred_storage_type)); + let blockstore_path = ledger_path.join(Self::blockstore_directory( + &options.advanced_options.shred_storage_type, + )); + let advanced_options = options.advanced_options.clone(); adjust_ulimit_nofile(options.enforce_ulimit_nofile)?; @@ -648,6 +664,7 @@ impl Blockstore { lowest_cleanup_slot: Arc::new(RwLock::new(0)), no_compaction: false, slots_stats: Arc::new(Mutex::new(SlotsStats::default())), + advanced_options, }; if initialize_transaction_status_index { blockstore.initialize_transaction_status_index()?; @@ -944,81 +961,101 @@ impl Blockstore { /// Collects and reports [`BlockstoreRocksDbColumnFamilyMetrics`] for the /// all the column families. pub fn submit_rocksdb_cf_metrics_for_all_cfs(&self) { + let advanced_options = &self.advanced_options; self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "slot_meta" + "slot_meta", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "dead_slots" + "dead_slots", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "duplicate_slots" + "duplicate_slots", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "erasure_meta" + "erasure_meta", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "orphans" + "orphans", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "bank_hash" + "bank_hash", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "root" + "root", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "index" + "index", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "shred_data" + "shred_data", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "shred_code" + "shred_code", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "transaction_status" + "transaction_status", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "address_signature" + "address_signature", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "transaction_memos" + "transaction_memos", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "transaction_status_index" + "transaction_status_index", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "rewards" + "rewards", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "blocktime" + "blocktime", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "perf_sample" + "perf_sample", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "block_height" + "block_height", + advanced_options )); self.submit_rocksdb_cf_metrics::(rocksdb_metric_header!( "blockstore_rocksdb_cfs", - "program_costs" + "program_costs", + advanced_options )); } @@ -4114,20 +4151,20 @@ pub fn create_new_ledger( genesis_config: &GenesisConfig, max_genesis_archive_unpacked_size: u64, access_type: AccessType, - shred_storage_type: ShredStorageType, + advanced_options: BlockstoreAdvancedOptions, ) -> Result { Blockstore::destroy(ledger_path)?; genesis_config.write(ledger_path)?; // Fill slot 0 with ticks that link back to the genesis_config to bootstrap the ledger. - let blockstore_dir = Blockstore::blockstore_directory(&shred_storage_type); + let blockstore_dir = Blockstore::blockstore_directory(&advanced_options.shred_storage_type); let blockstore = Blockstore::open_with_options( ledger_path, BlockstoreOptions { access_type, recovery_mode: None, enforce_ulimit_nofile: false, - shred_storage_type: shred_storage_type.clone(), + advanced_options: advanced_options.clone(), }, )?; let ticks_per_slot = genesis_config.ticks_per_slot; @@ -4297,7 +4334,7 @@ macro_rules! create_new_tmp_ledger { $crate::tmp_ledger_name!(), $genesis_config, $crate::blockstore_db::AccessType::PrimaryOnly, - $crate::blockstore_db::ShredStorageType::default(), + $crate::blockstore_db::BlockstoreAdvancedOptions::default(), ) }; } @@ -4309,7 +4346,7 @@ macro_rules! create_new_tmp_ledger_auto_delete { $crate::tmp_ledger_name!(), $genesis_config, $crate::blockstore_db::AccessType::PrimaryOnly, - $crate::blockstore_db::ShredStorageType::default(), + $crate::blockstore_db::BlockstoreAdvancedOptions::default(), ) }; } @@ -4321,9 +4358,11 @@ macro_rules! create_new_tmp_ledger_fifo_auto_delete { $crate::tmp_ledger_name!(), $genesis_config, $crate::blockstore_db::AccessType::PrimaryOnly, - $crate::blockstore_db::ShredStorageType::RocksFifo( - $crate::blockstore_db::BlockstoreRocksFifoOptions::default(), - ), + $crate::blockstore_db::BlockstoreAdvancedOptions { + shred_storage_type: $crate::blockstore_db::ShredStorageType::RocksFifo( + $crate::blockstore_db::BlockstoreRocksFifoOptions::default(), + ), + }, ) }; } @@ -4354,13 +4393,13 @@ pub fn create_new_ledger_from_name( name: &str, genesis_config: &GenesisConfig, access_type: AccessType, - shred_storage_type: ShredStorageType, + advanced_options: BlockstoreAdvancedOptions, ) -> (PathBuf, Hash) { let (ledger_path, blockhash) = create_new_ledger_from_name_auto_delete( name, genesis_config, access_type, - shred_storage_type, + advanced_options, ); (ledger_path.into_path(), blockhash) } @@ -4373,7 +4412,7 @@ pub fn create_new_ledger_from_name_auto_delete( name: &str, genesis_config: &GenesisConfig, access_type: AccessType, - shred_storage_type: ShredStorageType, + advanced_options: BlockstoreAdvancedOptions, ) -> (TempDir, Hash) { let ledger_path = get_ledger_path_from_name_auto_delete(name); let blockhash = create_new_ledger( @@ -4381,7 +4420,7 @@ pub fn create_new_ledger_from_name_auto_delete( genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, access_type, - shred_storage_type, + advanced_options, ) .unwrap(); (ledger_path, blockhash) @@ -4690,9 +4729,11 @@ pub mod tests { let blockstore = Blockstore::open_with_options( ledger_path.path(), BlockstoreOptions { - shred_storage_type: ShredStorageType::RocksFifo( - BlockstoreRocksFifoOptions::default(), - ), + advanced_options: BlockstoreAdvancedOptions { + shred_storage_type: ShredStorageType::RocksFifo( + BlockstoreRocksFifoOptions::default(), + ), + }, ..BlockstoreOptions::default() }, ) diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index ce69941938..29d6455d2e 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -411,12 +411,13 @@ impl Rocks { ) -> Vec { use columns::*; let access_type = &options.access_type; + let advanced_options = &options.advanced_options; let (cf_descriptor_shred_data, cf_descriptor_shred_code) = new_cf_descriptor_pair_shreds::( - &options.shred_storage_type, access_type, oldest_slot, + advanced_options, ); vec![ new_cf_descriptor::(access_type, oldest_slot), @@ -996,6 +997,23 @@ impl Default for ShredStorageType { } } +/// Advanced options for blockstore. +/// The each advanced option might also be used as a tag that supports +/// group-by operation when reporting Blockstore metrics. +#[derive(Clone)] +pub struct BlockstoreAdvancedOptions { + // Determine how to store both data and coding shreds. Default: RocksLevel. + pub shred_storage_type: ShredStorageType, +} + +impl Default for BlockstoreAdvancedOptions { + fn default() -> Self { + Self { + shred_storage_type: ShredStorageType::RocksLevel, + } + } +} + pub struct BlockstoreOptions { // The access type of blockstore. Default: PrimaryOnly pub access_type: AccessType, @@ -1003,8 +1021,7 @@ pub struct BlockstoreOptions { pub recovery_mode: Option, // Whether to allow unlimited number of open files. Default: true. pub enforce_ulimit_nofile: bool, - // Determine how to store both data and coding shreds. Default: RocksLevel. - pub shred_storage_type: ShredStorageType, + pub advanced_options: BlockstoreAdvancedOptions, } impl Default for BlockstoreOptions { @@ -1014,7 +1031,7 @@ impl Default for BlockstoreOptions { access_type: AccessType::PrimaryOnly, recovery_mode: None, enforce_ulimit_nofile: true, - shred_storage_type: ShredStorageType::RocksLevel, + advanced_options: BlockstoreAdvancedOptions::default(), } } } @@ -1445,11 +1462,11 @@ fn new_cf_descriptor_pair_shreds< D: 'static + Column + ColumnName, // Column Family for Data Shred C: 'static + Column + ColumnName, // Column Family for Coding Shred >( - shred_storage_type: &ShredStorageType, access_type: &AccessType, oldest_slot: &OldestSlot, + advanced_options: &BlockstoreAdvancedOptions, ) -> (ColumnFamilyDescriptor, ColumnFamilyDescriptor) { - match shred_storage_type { + match &advanced_options.shred_storage_type { ShredStorageType::RocksLevel => ( new_cf_descriptor::(access_type, oldest_slot), new_cf_descriptor::(access_type, oldest_slot), diff --git a/local-cluster/src/validator_configs.rs b/local-cluster/src/validator_configs.rs index 389317fa54..10a580d412 100644 --- a/local-cluster/src/validator_configs.rs +++ b/local-cluster/src/validator_configs.rs @@ -62,7 +62,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig { accounts_shrink_ratio: config.accounts_shrink_ratio, accounts_db_config: config.accounts_db_config.clone(), wait_to_vote_slot: config.wait_to_vote_slot, - shred_storage_type: config.shred_storage_type.clone(), + blockstore_advanced_options: config.blockstore_advanced_options.clone(), } } diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index 87d9b94703..14fe3d7415 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -13,7 +13,8 @@ use { socketaddr, }, solana_ledger::{ - blockstore::create_new_ledger, blockstore_db::ShredStorageType, create_new_tmp_ledger, + blockstore::create_new_ledger, blockstore_db::BlockstoreAdvancedOptions, + create_new_tmp_ledger, }, solana_net_utils::PortRange, solana_rpc::{rpc::JsonRpcConfig, rpc_pubsub_service::PubSubConfig}, @@ -582,7 +583,7 @@ impl TestValidator { .max_genesis_archive_unpacked_size .unwrap_or(MAX_GENESIS_ARCHIVE_UNPACKED_SIZE), solana_ledger::blockstore_db::AccessType::PrimaryOnly, - ShredStorageType::default(), + BlockstoreAdvancedOptions::default(), ) .map_err(|err| { format!( diff --git a/validator/src/main.rs b/validator/src/main.rs index c83c907124..dde2d6e9e4 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -33,8 +33,8 @@ use { contact_info::ContactInfo, }, solana_ledger::blockstore_db::{ - BlockstoreRecoveryMode, BlockstoreRocksFifoOptions, ShredStorageType, - DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES, + BlockstoreAdvancedOptions, BlockstoreRecoveryMode, BlockstoreRocksFifoOptions, + ShredStorageType, DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES, }, solana_perf::recycler::enable_recycler_warming, solana_poh::poh_service, @@ -2559,22 +2559,24 @@ pub fn main() { validator_config.max_ledger_shreds = Some(limit_ledger_size); } - validator_config.shred_storage_type = match matches.value_of("rocksdb_shred_compaction") { - None => ShredStorageType::default(), - Some(shred_compaction_string) => match shred_compaction_string { - "level" => ShredStorageType::RocksLevel, - "fifo" => { - let shred_storage_size = - value_t_or_exit!(matches, "rocksdb_fifo_shred_storage_size", u64); - ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions { - shred_data_cf_size: shred_storage_size / 2, - shred_code_cf_size: shred_storage_size / 2, - }) - } - _ => panic!( - "Unrecognized rocksdb-shred-compaction: {}", - shred_compaction_string - ), + validator_config.blockstore_advanced_options = BlockstoreAdvancedOptions { + shred_storage_type: match matches.value_of("rocksdb_shred_compaction") { + None => ShredStorageType::default(), + Some(shred_compaction_string) => match shred_compaction_string { + "level" => ShredStorageType::RocksLevel, + "fifo" => { + let shred_storage_size = + value_t_or_exit!(matches, "rocksdb_fifo_shred_storage_size", u64); + ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions { + shred_data_cf_size: shred_storage_size / 2, + shred_code_cf_size: shred_storage_size / 2, + }) + } + _ => panic!( + "Unrecognized rocksdb-shred-compaction: {}", + shred_compaction_string + ), + }, }, };