(LedgerStore) Include storage type as a tag in RocksDB metric reporting (#23523)

#### Summary of Changes
This PR further enables group by operation on storage type in blockstore_rocksdb_cfs metrics.
Such group-by allows us to further compare the performance metrics between rocks-level and
rocks-fifo.

To make things extensible, this PR introduces BlockstoreAdvancedOptions and move shred_storage_type. 
All fields in BlockstoreAdvancedOptions will support group-by operation in blockstore_rocksdb_cfs.

Dependency: #23580

core/src/validator.rs

@@ -36,7 +36,7 @@ use {
     solana_ledger::{
         bank_forks_utils,
         blockstore::{Blockstore, BlockstoreSignals, CompletedSlotsReceiver, PurgeType},
-        blockstore_db::{BlockstoreOptions, BlockstoreRecoveryMode, ShredStorageType},
+        blockstore_db::{BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRecoveryMode},
         blockstore_processor::{self, TransactionStatusSender},
         leader_schedule::FixedSchedule,
         leader_schedule_cache::LeaderScheduleCache,
@@ -165,7 +165,7 @@ pub struct ValidatorConfig {
     pub no_wait_for_vote_to_start_leader: bool,
     pub accounts_shrink_ratio: AccountShrinkThreshold,
     pub wait_to_vote_slot: Option<Slot>,
-    pub shred_storage_type: ShredStorageType,
+    pub blockstore_advanced_options: BlockstoreAdvancedOptions,
 }
 
 impl Default for ValidatorConfig {
@@ -226,7 +226,7 @@ impl Default for ValidatorConfig {
             accounts_shrink_ratio: AccountShrinkThreshold::default(),
             accounts_db_config: None,
             wait_to_vote_slot: None,
-            shred_storage_type: ShredStorageType::RocksLevel,
+            blockstore_advanced_options: BlockstoreAdvancedOptions::default(),
         }
     }
 }
@@ -1259,7 +1259,7 @@ fn new_banks_from_ledger(
         BlockstoreOptions {
             recovery_mode: config.wal_recovery_mode.clone(),
             enforce_ulimit_nofile,
-            shred_storage_type: config.shred_storage_type.clone(),
+            advanced_options: config.blockstore_advanced_options.clone(),
             ..BlockstoreOptions::default()
         },
     )

core/tests/ledger_cleanup.rs

@@ -9,7 +9,10 @@ mod tests {
         solana_core::ledger_cleanup_service::LedgerCleanupService,
         solana_ledger::{
             blockstore::{make_many_slot_shreds, Blockstore},
-            blockstore_db::{BlockstoreOptions, BlockstoreRocksFifoOptions, ShredStorageType},
+            blockstore_db::{
+                BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRocksFifoOptions,
+                ShredStorageType,
+            },
             get_tmp_ledger_path,
         },
         solana_measure::measure::Measure,
@@ -348,10 +351,14 @@ mod tests {
             &ledger_path,
             if config.fifo_compaction {
                 BlockstoreOptions {
-                    shred_storage_type: ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions {
+                    advanced_options: BlockstoreAdvancedOptions {
-                        shred_data_cf_size: config.shred_data_cf_size,
+                        shred_storage_type: ShredStorageType::RocksFifo(
-                        ..BlockstoreRocksFifoOptions::default()
+                            BlockstoreRocksFifoOptions {
-                    }),
+                                shred_data_cf_size: config.shred_data_cf_size,
+                                ..BlockstoreRocksFifoOptions::default()
+                            },
+                        ),
+                    },
                     ..BlockstoreOptions::default()
                 }
             } else {

genesis/src/main.rs

@@ -15,7 +15,7 @@ use {
     solana_genesis::{genesis_accounts::add_genesis_accounts, Base64Account},
     solana_ledger::{
         blockstore::create_new_ledger,
-        blockstore_db::{AccessType, ShredStorageType},
+        blockstore_db::{AccessType, BlockstoreAdvancedOptions},
     },
     solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
     solana_sdk::{
@@ -633,7 +633,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
         &genesis_config,
         max_genesis_archive_unpacked_size,
         AccessType::PrimaryOnly,
-        ShredStorageType::default(),
+        BlockstoreAdvancedOptions::default(),
     )?;
 
     println!("{}", genesis_config);

ledger-tool/src/main.rs

@@ -24,7 +24,8 @@ use {
         bank_forks_utils,
         blockstore::{create_new_ledger, Blockstore, PurgeType},
         blockstore_db::{
-            self, AccessType, BlockstoreOptions, BlockstoreRecoveryMode, Database, ShredStorageType,
+            self, AccessType, BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRecoveryMode,
+            Database,
         },
         blockstore_processor::ProcessOptions,
         shred::Shred,
@@ -1721,7 +1722,7 @@ fn main() {
                     &genesis_config,
                     solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
                     AccessType::PrimaryOnly,
-                    ShredStorageType::default(),
+                    BlockstoreAdvancedOptions::default(),
                 )
                 .unwrap_or_else(|err| {
                     eprintln!("Failed to write genesis config: {:?}", err);

ledger/src/blockstore.rs

@@ -5,8 +5,9 @@ use {
     crate::{
         ancestor_iterator::AncestorIterator,
         blockstore_db::{
-            columns as cf, AccessType, BlockstoreOptions, Column, ColumnName, Database,
+            columns as cf, AccessType, BlockstoreAdvancedOptions, BlockstoreOptions, Column,
-            IteratorDirection, IteratorMode, LedgerColumn, Result, ShredStorageType, WriteBatch,
+            ColumnName, Database, IteratorDirection, IteratorMode, LedgerColumn, Result,
+            ShredStorageType, WriteBatch,
         },
         blockstore_meta::*,
         leader_schedule_cache::LeaderScheduleCache,
@@ -177,6 +178,7 @@ pub struct Blockstore {
     pub lowest_cleanup_slot: Arc<RwLock<Slot>>,
     no_compaction: bool,
     slots_stats: Arc<Mutex<SlotsStats>>,
+    advanced_options: BlockstoreAdvancedOptions,
 }
 
 struct SlotsStats {
@@ -519,8 +521,20 @@ impl BlockstoreRocksDbColumnFamilyMetrics {
 }
 
 macro_rules! rocksdb_metric_header {
-    ($metric_name:literal, $cf_name:literal) => {
+    ($metric_name:literal, $cf_name:literal, $advanced_options:expr) => {
-        concat!($metric_name, ",cf_name=", $cf_name)
+        match $advanced_options.shred_storage_type {
+            ShredStorageType::RocksLevel =>
+                rocksdb_metric_header!(@all_fields $metric_name, $cf_name, "rocks_level"),
+            ShredStorageType::RocksFifo(_) =>
+                rocksdb_metric_header!(@all_fields $metric_name, $cf_name, "rocks_fifo"),
+        }
+    };
+
+    (@all_fields $metric_name:literal, $cf_name:literal, $storage_type:literal) => {
+        concat!($metric_name,
+            ",cf_name=", $cf_name,
+            ",storage=", $storage_type,
+        )
     };
 }
 use rocksdb_metric_header;
@@ -554,8 +568,10 @@ impl Blockstore {
 
     fn do_open(ledger_path: &Path, options: BlockstoreOptions) -> Result<Blockstore> {
         fs::create_dir_all(&ledger_path)?;
-        let blockstore_path =
+        let blockstore_path = ledger_path.join(Self::blockstore_directory(
-            ledger_path.join(Self::blockstore_directory(&options.shred_storage_type));
+            &options.advanced_options.shred_storage_type,
+        ));
+        let advanced_options = options.advanced_options.clone();
 
         adjust_ulimit_nofile(options.enforce_ulimit_nofile)?;
 
@@ -648,6 +664,7 @@ impl Blockstore {
             lowest_cleanup_slot: Arc::new(RwLock::new(0)),
             no_compaction: false,
             slots_stats: Arc::new(Mutex::new(SlotsStats::default())),
+            advanced_options,
         };
         if initialize_transaction_status_index {
             blockstore.initialize_transaction_status_index()?;
@@ -944,81 +961,101 @@ impl Blockstore {
     /// Collects and reports [`BlockstoreRocksDbColumnFamilyMetrics`] for the
     /// all the column families.
     pub fn submit_rocksdb_cf_metrics_for_all_cfs(&self) {
+        let advanced_options = &self.advanced_options;
         self.submit_rocksdb_cf_metrics::<cf::SlotMeta>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "slot_meta"
+            "slot_meta",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::DeadSlots>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "dead_slots"
+            "dead_slots",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::DuplicateSlots>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "duplicate_slots"
+            "duplicate_slots",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::ErasureMeta>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "erasure_meta"
+            "erasure_meta",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::Orphans>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "orphans"
+            "orphans",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::BankHash>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "bank_hash"
+            "bank_hash",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::Root>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "root"
+            "root",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::Index>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "index"
+            "index",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::ShredData>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "shred_data"
+            "shred_data",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::ShredCode>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "shred_code"
+            "shred_code",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::TransactionStatus>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "transaction_status"
+            "transaction_status",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::AddressSignatures>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "address_signature"
+            "address_signature",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::TransactionMemos>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "transaction_memos"
+            "transaction_memos",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::TransactionStatusIndex>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "transaction_status_index"
+            "transaction_status_index",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::Rewards>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "rewards"
+            "rewards",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::Blocktime>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "blocktime"
+            "blocktime",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::PerfSamples>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "perf_sample"
+            "perf_sample",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::BlockHeight>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "block_height"
+            "block_height",
+            advanced_options
         ));
         self.submit_rocksdb_cf_metrics::<cf::ProgramCosts>(rocksdb_metric_header!(
             "blockstore_rocksdb_cfs",
-            "program_costs"
+            "program_costs",
+            advanced_options
         ));
     }
 
@@ -4114,20 +4151,20 @@ pub fn create_new_ledger(
     genesis_config: &GenesisConfig,
     max_genesis_archive_unpacked_size: u64,
     access_type: AccessType,
-    shred_storage_type: ShredStorageType,
+    advanced_options: BlockstoreAdvancedOptions,
 ) -> Result<Hash> {
     Blockstore::destroy(ledger_path)?;
     genesis_config.write(ledger_path)?;
 
     // Fill slot 0 with ticks that link back to the genesis_config to bootstrap the ledger.
-    let blockstore_dir = Blockstore::blockstore_directory(&shred_storage_type);
+    let blockstore_dir = Blockstore::blockstore_directory(&advanced_options.shred_storage_type);
     let blockstore = Blockstore::open_with_options(
         ledger_path,
         BlockstoreOptions {
             access_type,
             recovery_mode: None,
             enforce_ulimit_nofile: false,
-            shred_storage_type: shred_storage_type.clone(),
+            advanced_options: advanced_options.clone(),
         },
     )?;
     let ticks_per_slot = genesis_config.ticks_per_slot;
@@ -4297,7 +4334,7 @@ macro_rules! create_new_tmp_ledger {
             $crate::tmp_ledger_name!(),
             $genesis_config,
             $crate::blockstore_db::AccessType::PrimaryOnly,
-            $crate::blockstore_db::ShredStorageType::default(),
+            $crate::blockstore_db::BlockstoreAdvancedOptions::default(),
         )
     };
 }
@@ -4309,7 +4346,7 @@ macro_rules! create_new_tmp_ledger_auto_delete {
             $crate::tmp_ledger_name!(),
             $genesis_config,
             $crate::blockstore_db::AccessType::PrimaryOnly,
-            $crate::blockstore_db::ShredStorageType::default(),
+            $crate::blockstore_db::BlockstoreAdvancedOptions::default(),
         )
     };
 }
@@ -4321,9 +4358,11 @@ macro_rules! create_new_tmp_ledger_fifo_auto_delete {
             $crate::tmp_ledger_name!(),
             $genesis_config,
             $crate::blockstore_db::AccessType::PrimaryOnly,
-            $crate::blockstore_db::ShredStorageType::RocksFifo(
+            $crate::blockstore_db::BlockstoreAdvancedOptions {
-                $crate::blockstore_db::BlockstoreRocksFifoOptions::default(),
+                shred_storage_type: $crate::blockstore_db::ShredStorageType::RocksFifo(
-            ),
+                    $crate::blockstore_db::BlockstoreRocksFifoOptions::default(),
+                ),
+            },
         )
     };
 }
@@ -4354,13 +4393,13 @@ pub fn create_new_ledger_from_name(
     name: &str,
     genesis_config: &GenesisConfig,
     access_type: AccessType,
-    shred_storage_type: ShredStorageType,
+    advanced_options: BlockstoreAdvancedOptions,
 ) -> (PathBuf, Hash) {
     let (ledger_path, blockhash) = create_new_ledger_from_name_auto_delete(
         name,
         genesis_config,
         access_type,
-        shred_storage_type,
+        advanced_options,
     );
     (ledger_path.into_path(), blockhash)
 }
@@ -4373,7 +4412,7 @@ pub fn create_new_ledger_from_name_auto_delete(
     name: &str,
     genesis_config: &GenesisConfig,
     access_type: AccessType,
-    shred_storage_type: ShredStorageType,
+    advanced_options: BlockstoreAdvancedOptions,
 ) -> (TempDir, Hash) {
     let ledger_path = get_ledger_path_from_name_auto_delete(name);
     let blockhash = create_new_ledger(
@@ -4381,7 +4420,7 @@ pub fn create_new_ledger_from_name_auto_delete(
         genesis_config,
         MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
         access_type,
-        shred_storage_type,
+        advanced_options,
     )
     .unwrap();
     (ledger_path, blockhash)
@@ -4690,9 +4729,11 @@ pub mod tests {
         let blockstore = Blockstore::open_with_options(
             ledger_path.path(),
             BlockstoreOptions {
-                shred_storage_type: ShredStorageType::RocksFifo(
+                advanced_options: BlockstoreAdvancedOptions {
-                    BlockstoreRocksFifoOptions::default(),
+                    shred_storage_type: ShredStorageType::RocksFifo(
-                ),
+                        BlockstoreRocksFifoOptions::default(),
+                    ),
+                },
                 ..BlockstoreOptions::default()
             },
         )

ledger/src/blockstore_db.rs

@@ -411,12 +411,13 @@ impl Rocks {
     ) -> Vec<ColumnFamilyDescriptor> {
         use columns::*;
         let access_type = &options.access_type;
+        let advanced_options = &options.advanced_options;
 
         let (cf_descriptor_shred_data, cf_descriptor_shred_code) =
             new_cf_descriptor_pair_shreds::<ShredData, ShredCode>(
-                &options.shred_storage_type,
                 access_type,
                 oldest_slot,
+                advanced_options,
             );
         vec![
             new_cf_descriptor::<SlotMeta>(access_type, oldest_slot),
@@ -996,6 +997,23 @@ impl Default for ShredStorageType {
     }
 }
 
+/// Advanced options for blockstore.
+/// The each advanced option might also be used as a tag that supports
+/// group-by operation when reporting Blockstore metrics.
+#[derive(Clone)]
+pub struct BlockstoreAdvancedOptions {
+    // Determine how to store both data and coding shreds. Default: RocksLevel.
+    pub shred_storage_type: ShredStorageType,
+}
+
+impl Default for BlockstoreAdvancedOptions {
+    fn default() -> Self {
+        Self {
+            shred_storage_type: ShredStorageType::RocksLevel,
+        }
+    }
+}
+
 pub struct BlockstoreOptions {
     // The access type of blockstore. Default: PrimaryOnly
     pub access_type: AccessType,
@@ -1003,8 +1021,7 @@ pub struct BlockstoreOptions {
     pub recovery_mode: Option<BlockstoreRecoveryMode>,
     // Whether to allow unlimited number of open files. Default: true.
     pub enforce_ulimit_nofile: bool,
-    // Determine how to store both data and coding shreds. Default: RocksLevel.
+    pub advanced_options: BlockstoreAdvancedOptions,
-    pub shred_storage_type: ShredStorageType,
 }
 
 impl Default for BlockstoreOptions {
@@ -1014,7 +1031,7 @@ impl Default for BlockstoreOptions {
             access_type: AccessType::PrimaryOnly,
             recovery_mode: None,
             enforce_ulimit_nofile: true,
-            shred_storage_type: ShredStorageType::RocksLevel,
+            advanced_options: BlockstoreAdvancedOptions::default(),
         }
     }
 }
@@ -1445,11 +1462,11 @@ fn new_cf_descriptor_pair_shreds<
     D: 'static + Column + ColumnName, // Column Family for Data Shred
     C: 'static + Column + ColumnName, // Column Family for Coding Shred
 >(
-    shred_storage_type: &ShredStorageType,
     access_type: &AccessType,
     oldest_slot: &OldestSlot,
+    advanced_options: &BlockstoreAdvancedOptions,
 ) -> (ColumnFamilyDescriptor, ColumnFamilyDescriptor) {
-    match shred_storage_type {
+    match &advanced_options.shred_storage_type {
         ShredStorageType::RocksLevel => (
             new_cf_descriptor::<D>(access_type, oldest_slot),
             new_cf_descriptor::<C>(access_type, oldest_slot),

local-cluster/src/validator_configs.rs

@@ -62,7 +62,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig {
         accounts_shrink_ratio: config.accounts_shrink_ratio,
         accounts_db_config: config.accounts_db_config.clone(),
         wait_to_vote_slot: config.wait_to_vote_slot,
-        shred_storage_type: config.shred_storage_type.clone(),
+        blockstore_advanced_options: config.blockstore_advanced_options.clone(),
     }
 }
 

test-validator/src/lib.rs

@@ -13,7 +13,8 @@ use {
         socketaddr,
     },
     solana_ledger::{
-        blockstore::create_new_ledger, blockstore_db::ShredStorageType, create_new_tmp_ledger,
+        blockstore::create_new_ledger, blockstore_db::BlockstoreAdvancedOptions,
+        create_new_tmp_ledger,
     },
     solana_net_utils::PortRange,
     solana_rpc::{rpc::JsonRpcConfig, rpc_pubsub_service::PubSubConfig},
@@ -582,7 +583,7 @@ impl TestValidator {
                         .max_genesis_archive_unpacked_size
                         .unwrap_or(MAX_GENESIS_ARCHIVE_UNPACKED_SIZE),
                     solana_ledger::blockstore_db::AccessType::PrimaryOnly,
-                    ShredStorageType::default(),
+                    BlockstoreAdvancedOptions::default(),
                 )
                 .map_err(|err| {
                     format!(

validator/src/main.rs

@@ -33,8 +33,8 @@ use {
         contact_info::ContactInfo,
     },
     solana_ledger::blockstore_db::{
-        BlockstoreRecoveryMode, BlockstoreRocksFifoOptions, ShredStorageType,
+        BlockstoreAdvancedOptions, BlockstoreRecoveryMode, BlockstoreRocksFifoOptions,
-        DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
+        ShredStorageType, DEFAULT_ROCKS_FIFO_SHRED_STORAGE_SIZE_BYTES,
     },
     solana_perf::recycler::enable_recycler_warming,
     solana_poh::poh_service,
@@ -2559,22 +2559,24 @@ pub fn main() {
         validator_config.max_ledger_shreds = Some(limit_ledger_size);
     }
 
-    validator_config.shred_storage_type = match matches.value_of("rocksdb_shred_compaction") {
+    validator_config.blockstore_advanced_options = BlockstoreAdvancedOptions {
-        None => ShredStorageType::default(),
+        shred_storage_type: match matches.value_of("rocksdb_shred_compaction") {
-        Some(shred_compaction_string) => match shred_compaction_string {
+            None => ShredStorageType::default(),
-            "level" => ShredStorageType::RocksLevel,
+            Some(shred_compaction_string) => match shred_compaction_string {
-            "fifo" => {
+                "level" => ShredStorageType::RocksLevel,
-                let shred_storage_size =
+                "fifo" => {
-                    value_t_or_exit!(matches, "rocksdb_fifo_shred_storage_size", u64);
+                    let shred_storage_size =
-                ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions {
+                        value_t_or_exit!(matches, "rocksdb_fifo_shred_storage_size", u64);
-                    shred_data_cf_size: shred_storage_size / 2,
+                    ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions {
-                    shred_code_cf_size: shred_storage_size / 2,
+                        shred_data_cf_size: shred_storage_size / 2,
-                })
+                        shred_code_cf_size: shred_storage_size / 2,
-            }
+                    })
-            _ => panic!(
+                }
-                "Unrecognized rocksdb-shred-compaction: {}",
+                _ => panic!(
-                shred_compaction_string
+                    "Unrecognized rocksdb-shred-compaction: {}",
-            ),
+                    shred_compaction_string
+                ),
+            },
         },
     };