(LedgerStore) Include storage type as a tag in RocksDB metric reporting (#23523)

#### Summary of Changes
This PR further enables group by operation on storage type in blockstore_rocksdb_cfs metrics.
Such group-by allows us to further compare the performance metrics between rocks-level and
rocks-fifo.

To make things extensible, this PR introduces BlockstoreAdvancedOptions and move shred_storage_type. 
All fields in BlockstoreAdvancedOptions will support group-by operation in blockstore_rocksdb_cfs.

Dependency: #23580

core/src/validator.rs

@@ -36,7 +36,7 @@ use {
     solana_ledger::{
         bank_forks_utils,
         blockstore::{Blockstore, BlockstoreSignals, CompletedSlotsReceiver, PurgeType},
-        blockstore_db::{BlockstoreOptions, BlockstoreRecoveryMode, ShredStorageType},
+        blockstore_db::{BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRecoveryMode},
         blockstore_processor::{self, TransactionStatusSender},
         leader_schedule::FixedSchedule,
         leader_schedule_cache::LeaderScheduleCache,
@@ -165,7 +165,7 @@ pub struct ValidatorConfig {
     pub no_wait_for_vote_to_start_leader: bool,
     pub accounts_shrink_ratio: AccountShrinkThreshold,
     pub wait_to_vote_slot: Option<Slot>,
-    pub shred_storage_type: ShredStorageType,
+    pub blockstore_advanced_options: BlockstoreAdvancedOptions,
 }
 
 impl Default for ValidatorConfig {
@@ -226,7 +226,7 @@ impl Default for ValidatorConfig {
             accounts_shrink_ratio: AccountShrinkThreshold::default(),
             accounts_db_config: None,
             wait_to_vote_slot: None,
-            shred_storage_type: ShredStorageType::RocksLevel,
+            blockstore_advanced_options: BlockstoreAdvancedOptions::default(),
         }
     }
 }
@@ -1259,7 +1259,7 @@ fn new_banks_from_ledger(
         BlockstoreOptions {
             recovery_mode: config.wal_recovery_mode.clone(),
             enforce_ulimit_nofile,
-            shred_storage_type: config.shred_storage_type.clone(),
+            advanced_options: config.blockstore_advanced_options.clone(),
             ..BlockstoreOptions::default()
         },
     )

core/tests/ledger_cleanup.rs

@@ -9,7 +9,10 @@ mod tests {
         solana_core::ledger_cleanup_service::LedgerCleanupService,
         solana_ledger::{
             blockstore::{make_many_slot_shreds, Blockstore},
-            blockstore_db::{BlockstoreOptions, BlockstoreRocksFifoOptions, ShredStorageType},
+            blockstore_db::{
+                BlockstoreAdvancedOptions, BlockstoreOptions, BlockstoreRocksFifoOptions,
+                ShredStorageType,
+            },
             get_tmp_ledger_path,
         },
         solana_measure::measure::Measure,
@@ -348,10 +351,14 @@ mod tests {
             &ledger_path,
             if config.fifo_compaction {
                 BlockstoreOptions {
-                    shred_storage_type: ShredStorageType::RocksFifo(BlockstoreRocksFifoOptions {
-                        shred_data_cf_size: config.shred_data_cf_size,
-                        ..BlockstoreRocksFifoOptions::default()
-                    }),
+                    advanced_options: BlockstoreAdvancedOptions {
+                        shred_storage_type: ShredStorageType::RocksFifo(
+                            BlockstoreRocksFifoOptions {
+                                shred_data_cf_size: config.shred_data_cf_size,
+                                ..BlockstoreRocksFifoOptions::default()
+                            },
+                        ),
+                    },
                     ..BlockstoreOptions::default()
                 }
             } else {