diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index c5311a830e..a237e0b9e1 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -33,8 +33,13 @@ use { }, solana_measure::measure::Measure, solana_runtime::{ +<<<<<<< HEAD accounts_db::AccountsDbConfig, accounts_index::{AccountsIndexConfig, ScanConfig}, +======= + accounts_db::{AccountsDbConfig, FillerAccountsConfig}, + accounts_index::{AccountsIndexConfig, IndexLimitMb, ScanConfig}, +>>>>>>> 1bc49d219 (IndexLimitMb option adds 'Unspecified' state (#24249)) bank::{Bank, RewardCalculationEvent}, bank_forks::BankForks, cost_model::CostModel, @@ -1997,13 +2002,15 @@ fn main() { let system_monitor_service = SystemMonitorService::new(Arc::clone(&exit_signal), true, false); - if let Some(limit) = + accounts_index_config.index_limit_mb = if let Some(limit) = value_t!(arg_matches, "accounts_index_memory_limit_mb", usize).ok() { - accounts_index_config.index_limit_mb = Some(limit); + IndexLimitMb::Limit(limit) } else if arg_matches.is_present("disable_accounts_disk_index") { - accounts_index_config.index_limit_mb = None; - } + IndexLimitMb::InMemOnly + } else { + IndexLimitMb::Unspecified + }; { let mut accounts_index_paths: Vec = diff --git a/runtime/src/accounts_index.rs b/runtime/src/accounts_index.rs index bbcb1f47bf..1e1ce4527c 100644 --- a/runtime/src/accounts_index.rs +++ b/runtime/src/accounts_index.rs @@ -50,7 +50,7 @@ pub const ACCOUNTS_INDEX_CONFIG_FOR_TESTING: AccountsIndexConfig = AccountsIndex bins: Some(BINS_FOR_TESTING), flush_threads: Some(FLUSH_THREADS_TESTING), drives: None, - index_limit_mb: None, + index_limit_mb: IndexLimitMb::Unspecified, ages_to_stay_in_cache: None, scan_results_limit_bytes: None, started_from_validator: false, @@ -59,7 +59,7 @@ pub const ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS: AccountsIndexConfig = AccountsIn bins: Some(BINS_FOR_BENCHMARKS), flush_threads: Some(FLUSH_THREADS_TESTING), drives: None, - index_limit_mb: None, + index_limit_mb: IndexLimitMb::Unspecified, ages_to_stay_in_cache: None, scan_results_limit_bytes: None, started_from_validator: false, @@ -157,12 +157,29 @@ pub struct AccountSecondaryIndexesIncludeExclude { pub keys: HashSet, } +/// specification of how much memory in-mem portion of account index can use +#[derive(Debug, Clone)] +pub enum IndexLimitMb { + /// nothing explicit specified, so default + Unspecified, + /// limit was specified, use disk index for rest + Limit(usize), + /// in-mem-only was specified, no disk index + InMemOnly, +} + +impl Default for IndexLimitMb { + fn default() -> Self { + Self::Unspecified + } +} + #[derive(Debug, Default, Clone)] pub struct AccountsIndexConfig { pub bins: Option, pub flush_threads: Option, pub drives: Option>, - pub index_limit_mb: Option, + pub index_limit_mb: IndexLimitMb, pub ages_to_stay_in_cache: Option, pub scan_results_limit_bytes: Option, /// true if the accounts index is being created as a result of being started as a validator (as opposed to test, etc.) diff --git a/runtime/src/bucket_map_holder.rs b/runtime/src/bucket_map_holder.rs index 87b2ee26fe..23ab83f2d9 100644 --- a/runtime/src/bucket_map_holder.rs +++ b/runtime/src/bucket_map_holder.rs @@ -1,6 +1,6 @@ use { crate::{ - accounts_index::{AccountsIndexConfig, IndexValue}, + accounts_index::{AccountsIndexConfig, IndexLimitMb, IndexValue}, bucket_map_holder_stats::BucketMapHolderStats, in_mem_accounts_index::{InMemAccountsIndex, SlotT}, waitable_condvar::WaitableCondvar, @@ -153,7 +153,13 @@ impl BucketMapHolder { let mut bucket_config = BucketMapConfig::new(bins); bucket_config.drives = config.as_ref().and_then(|config| config.drives.clone()); - let mut mem_budget_mb = config.as_ref().and_then(|config| config.index_limit_mb); + let mut mem_budget_mb = config.as_ref().and_then(|config| { + if let IndexLimitMb::Limit(mb) = config.index_limit_mb { + Some(mb) + } else { + None + } + }); let bucket_map_tests_allowed = mem_budget_mb.is_none() && !config .as_ref() @@ -391,7 +397,7 @@ pub mod tests { fn test_disk_index_enabled() { let bins = 1; let config = AccountsIndexConfig { - index_limit_mb: Some(0), + index_limit_mb: IndexLimitMb::Limit(0), ..AccountsIndexConfig::default() }; let test = BucketMapHolder::::new(bins, &Some(config), 1); diff --git a/runtime/src/in_mem_accounts_index.rs b/runtime/src/in_mem_accounts_index.rs index ab155d3d83..85bb34be6b 100644 --- a/runtime/src/in_mem_accounts_index.rs +++ b/runtime/src/in_mem_accounts_index.rs @@ -1212,7 +1212,7 @@ impl Drop for FlushGuard<'_> { mod tests { use { super::*, - crate::accounts_index::{AccountsIndexConfig, BINS_FOR_TESTING}, + crate::accounts_index::{AccountsIndexConfig, IndexLimitMb, BINS_FOR_TESTING}, itertools::Itertools, }; @@ -1230,7 +1230,7 @@ mod tests { let holder = Arc::new(BucketMapHolder::new( BINS_FOR_TESTING, &Some(AccountsIndexConfig { - index_limit_mb: Some(1), + index_limit_mb: IndexLimitMb::Limit(1), ..AccountsIndexConfig::default() }), 1, diff --git a/validator/src/main.rs b/validator/src/main.rs index 6c94650779..d16c508b9a 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -46,7 +46,7 @@ use { }, accounts_index::{ AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude, - AccountsIndexConfig, + AccountsIndexConfig, IndexLimitMb, }, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, snapshot_config::SnapshotConfig, @@ -2194,11 +2194,14 @@ pub fn main() { accounts_index_config.bins = Some(bins); } - if let Some(limit) = value_t!(matches, "accounts_index_memory_limit_mb", usize).ok() { - accounts_index_config.index_limit_mb = Some(limit); - } else if matches.is_present("disable_accounts_disk_index") { - accounts_index_config.index_limit_mb = None; - } + accounts_index_config.index_limit_mb = + if let Some(limit) = value_t!(matches, "accounts_index_memory_limit_mb", usize).ok() { + IndexLimitMb::Limit(limit) + } else if matches.is_present("disable_accounts_disk_index") { + IndexLimitMb::InMemOnly + } else { + IndexLimitMb::Unspecified + }; { let mut accounts_index_paths: Vec = if matches.is_present("accounts_index_path") {