plumb more accounts_index bins (#19123)

core/tests/snapshots.rs

@@ -191,6 +191,7 @@ mod tests {
             check_hash_calculation,
             false,
             false,
+            solana_runtime::accounts_index::BINS_FOR_TESTING,
         )
         .unwrap();
 
@@ -807,6 +808,7 @@ mod tests {
             false,
             false,
             false,
+            solana_runtime::accounts_index::BINS_FOR_TESTING,
         )?;
 
         assert_eq!(bank, &deserialized_bank);

ledger/src/bank_forks_utils.rs

@@ -132,6 +132,7 @@ fn load_from_snapshot(
         process_options.accounts_db_test_hash_calculation,
         process_options.accounts_db_skip_shrink,
         process_options.verify_index,
+        solana_runtime::accounts_index::BINS_DEFAULT,
     )
     .expect("Load from snapshot failed");
 

ledger/src/blockstore_processor.rs

@@ -416,6 +416,7 @@ pub fn process_blockstore(
         opts.accounts_db_caching_enabled,
         opts.shrink_ratio,
         false,
+        solana_runtime::accounts_index::BINS_DEFAULT,
     );
     let bank0 = Arc::new(bank0);
     info!("processing ledger for slot 0...");

replica-node/src/replica_node.rs

@@ -127,6 +127,7 @@ fn initialize_from_snapshot(
         process_options.accounts_db_test_hash_calculation,
         false,
         process_options.verify_index,
+        solana_runtime::accounts_index::BINS_DEFAULT,
     )
     .unwrap();
 

runtime/src/accounts.rs

@@ -3,7 +3,9 @@ use crate::{
         AccountShrinkThreshold, AccountsDb, BankHashInfo, ErrorCounters, LoadHint, LoadedAccount,
         ScanStorageResult,
     },
-    accounts_index::{AccountSecondaryIndexes, IndexKey, ScanResult},
+    accounts_index::{
+        AccountSecondaryIndexes, IndexKey, ScanResult, BINS_FOR_BENCHMARKS, BINS_FOR_TESTING,
+    },
     ancestors::Ancestors,
     bank::{
         NonceRollbackFull, NonceRollbackInfo, RentDebits, TransactionCheckResult,
@@ -131,13 +133,13 @@ impl Accounts {
         caching_enabled: bool,
         shrink_ratio: AccountShrinkThreshold,
     ) -> Self {
-        // will diverge
         Self::new_with_config(
             paths,
             cluster_type,
             account_indexes,
             caching_enabled,
             shrink_ratio,
+            BINS_FOR_TESTING,
         )
     }
 
@@ -148,13 +150,13 @@ impl Accounts {
         caching_enabled: bool,
         shrink_ratio: AccountShrinkThreshold,
     ) -> Self {
-        // will diverge
         Self::new_with_config(
             paths,
             cluster_type,
             account_indexes,
             caching_enabled,
             shrink_ratio,
+            BINS_FOR_BENCHMARKS,
         )
     }
 
@@ -164,6 +166,7 @@ impl Accounts {
         account_indexes: AccountSecondaryIndexes,
         caching_enabled: bool,
         shrink_ratio: AccountShrinkThreshold,
+        accounts_index_bins: usize,
     ) -> Self {
         Self {
             accounts_db: Arc::new(AccountsDb::new_with_config(
@@ -172,6 +175,7 @@ impl Accounts {
                 account_indexes,
                 caching_enabled,
                 shrink_ratio,
+                accounts_index_bins,
             )),
             account_locks: Mutex::new(AccountLocks::default()),
         }

runtime/src/accounts_db.rs

@@ -24,7 +24,7 @@ use crate::{
     accounts_hash::{AccountsHash, CalculateHashIntermediate, HashStats, PreviousPass},
     accounts_index::{
         AccountIndexGetResult, AccountSecondaryIndexes, AccountsIndex, AccountsIndexRootsStats,
-        IndexKey, IsCached, ScanResult, SlotList, SlotSlice, ZeroLamport, BINS_DEFAULT,
+        IndexKey, IsCached, ScanResult, SlotList, SlotSlice, ZeroLamport, BINS_FOR_TESTING,
     },
     ancestors::Ancestors,
     append_vec::{AppendVec, StoredAccountMeta, StoredMeta, StoredMetaWriteVersion},
@@ -1413,13 +1413,13 @@ impl AccountsDb {
     }
 
     pub fn new_for_tests(paths: Vec<PathBuf>, cluster_type: &ClusterType) -> Self {
-        // will diverge
         AccountsDb::new_with_config(
             paths,
             cluster_type,
             AccountSecondaryIndexes::default(),
             false,
             AccountShrinkThreshold::default(),
+            BINS_FOR_TESTING,
         )
     }
 
@@ -1429,8 +1429,9 @@ impl AccountsDb {
         account_indexes: AccountSecondaryIndexes,
         caching_enabled: bool,
         shrink_ratio: AccountShrinkThreshold,
+        accounts_index_bins: usize,
     ) -> Self {
-        let accounts_index = AccountsIndex::new(BINS_DEFAULT);
+        let accounts_index = AccountsIndex::new(accounts_index_bins);
         let mut new = if !paths.is_empty() {
             Self {
                 paths,
@@ -6232,6 +6233,7 @@ impl AccountsDb {
             account_indexes,
             caching_enabled,
             shrink_ratio,
+            BINS_FOR_TESTING,
         )
     }
 

runtime/src/accounts_index.rs

@@ -33,7 +33,8 @@ use thiserror::Error;
 
 pub const ITER_BATCH_SIZE: usize = 1000;
 pub const BINS_DEFAULT: usize = 16;
-const BINS_FOR_TESTING: usize = BINS_DEFAULT;
+pub const BINS_FOR_TESTING: usize = BINS_DEFAULT;
+pub const BINS_FOR_BENCHMARKS: usize = BINS_DEFAULT;
 pub type ScanResult<T> = Result<T, ScanError>;
 pub type SlotList<T> = Vec<(Slot, T)>;
 pub type SlotSlice<'s, T> = &'s [(Slot, T)];

runtime/src/bank.rs

@@ -39,7 +39,9 @@ use crate::{
         TransactionLoaders,
     },
     accounts_db::{AccountShrinkThreshold, ErrorCounters, SnapshotStorage, SnapshotStorages},
-    accounts_index::{AccountSecondaryIndexes, IndexKey, ScanResult},
+    accounts_index::{
+        AccountSecondaryIndexes, IndexKey, ScanResult, BINS_FOR_BENCHMARKS, BINS_FOR_TESTING,
+    },
     ancestors::{Ancestors, AncestorsForSerialization},
     blockhash_queue::BlockhashQueue,
     builtins::{self, ActivationType},
@@ -1188,6 +1190,7 @@ impl Bank {
             accounts_db_caching_enabled,
             shrink_ratio,
             debug_do_not_add_builtins,
+            BINS_FOR_TESTING,
         )
     }
 
@@ -1212,9 +1215,11 @@ impl Bank {
             accounts_db_caching_enabled,
             shrink_ratio,
             debug_do_not_add_builtins,
+            BINS_FOR_BENCHMARKS,
         )
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub fn new_with_paths(
         genesis_config: &GenesisConfig,
         paths: Vec<PathBuf>,
@@ -1225,6 +1230,7 @@ impl Bank {
         accounts_db_caching_enabled: bool,
         shrink_ratio: AccountShrinkThreshold,
         debug_do_not_add_builtins: bool,
+        accounts_index_bins: usize,
     ) -> Self {
         let accounts = Accounts::new_with_config(
             paths,
@@ -1232,6 +1238,7 @@ impl Bank {
             account_indexes,
             accounts_db_caching_enabled,
             shrink_ratio,
+            accounts_index_bins,
         );
         let mut bank = Self::default_with_accounts(accounts);
         bank.ancestors = Ancestors::from(vec![bank.slot()]);

runtime/src/serde_snapshot.rs

@@ -197,6 +197,7 @@ pub(crate) fn bank_from_streams<R>(
     limit_load_slot_count_from_snapshot: Option<usize>,
     shrink_ratio: AccountShrinkThreshold,
     verify_index: bool,
+    accounts_index_bins: usize,
 ) -> std::result::Result<Bank, Error>
 where
     R: Read,
@@ -234,6 +235,7 @@ where
                 limit_load_slot_count_from_snapshot,
                 shrink_ratio,
                 verify_index,
+                accounts_index_bins,
             )?;
             Ok(bank)
         }};
@@ -326,6 +328,7 @@ fn reconstruct_bank_from_fields<E>(
     limit_load_slot_count_from_snapshot: Option<usize>,
     shrink_ratio: AccountShrinkThreshold,
     verify_index: bool,
+    accounts_index_bins: usize,
 ) -> Result<Bank, Error>
 where
     E: SerializableStorage + std::marker::Sync,
@@ -340,6 +343,7 @@ where
         limit_load_slot_count_from_snapshot,
         shrink_ratio,
         verify_index,
+        accounts_index_bins,
     )?;
     accounts_db.freeze_accounts(
         &Ancestors::from(&bank_fields.ancestors),
@@ -380,6 +384,7 @@ where
     Ok(())
 }
 
+#[allow(clippy::too_many_arguments)]
 fn reconstruct_accountsdb_from_fields<E>(
     snapshot_accounts_db_fields: SnapshotAccountsDbFields<E>,
     account_paths: &[PathBuf],
@@ -390,6 +395,7 @@ fn reconstruct_accountsdb_from_fields<E>(
     limit_load_slot_count_from_snapshot: Option<usize>,
     shrink_ratio: AccountShrinkThreshold,
     verify_index: bool,
+    accounts_index_bins: usize,
 ) -> Result<AccountsDb, Error>
 where
     E: SerializableStorage + std::marker::Sync,
@@ -400,6 +406,7 @@ where
         account_secondary_indexes,
         caching_enabled,
         shrink_ratio,
+        accounts_index_bins,
     );
 
     let AccountsDbFields(

runtime/src/serde_snapshot/tests.rs

@@ -81,6 +81,7 @@ where
         None,
         AccountShrinkThreshold::default(),
         false,
+        crate::accounts_index::BINS_FOR_TESTING,
     )
 }
 
@@ -242,6 +243,7 @@ fn test_bank_serialize_style(serde_style: SerdeStyle) {
         None,
         AccountShrinkThreshold::default(),
         false,
+        crate::accounts_index::BINS_FOR_TESTING,
     )
     .unwrap();
     dbank.src = ref_sc;

runtime/src/snapshot_utils.rs

@@ -726,6 +726,7 @@ pub fn bank_from_snapshot_archives(
     test_hash_calculation: bool,
     accounts_db_skip_shrink: bool,
     verify_index: bool,
+    accounts_index_bins: usize,
 ) -> Result<(Bank, BankFromArchiveTimings)> {
     check_are_snapshots_compatible(
         full_snapshot_archive_info,
@@ -789,6 +790,7 @@ pub fn bank_from_snapshot_archives(
         limit_load_slot_count_from_snapshot,
         shrink_ratio,
         verify_index,
+        accounts_index_bins,
     )?;
     measure_rebuild.stop();
     info!("{}", measure_rebuild);
@@ -831,6 +833,7 @@ pub fn bank_from_latest_snapshot_archives(
     test_hash_calculation: bool,
     accounts_db_skip_shrink: bool,
     verify_index: bool,
+    accounts_index_bins: usize,
 ) -> Result<(Bank, BankFromArchiveTimings)> {
     let full_snapshot_archive_info = get_highest_full_snapshot_archive_info(&snapshot_archives_dir)
         .ok_or(SnapshotError::NoSnapshotArchives)?;
@@ -868,6 +871,7 @@ pub fn bank_from_latest_snapshot_archives(
         test_hash_calculation,
         accounts_db_skip_shrink,
         verify_index,
+        accounts_index_bins,
     )?;
 
     verify_bank_against_expected_slot_hash(
@@ -1367,6 +1371,7 @@ fn rebuild_bank_from_snapshots(
     limit_load_slot_count_from_snapshot: Option<usize>,
     shrink_ratio: AccountShrinkThreshold,
     verify_index: bool,
+    accounts_index_bins: usize,
 ) -> Result<Bank> {
     let (full_snapshot_version, full_snapshot_root_paths) =
         verify_unpacked_snapshots_dir_and_version(
@@ -1414,6 +1419,7 @@ fn rebuild_bank_from_snapshots(
                     limit_load_slot_count_from_snapshot,
                     shrink_ratio,
                     verify_index,
+                    accounts_index_bins,
                 ),
             }?,
         )
@@ -2484,6 +2490,7 @@ mod tests {
             false,
             false,
             false,
+            crate::accounts_index::BINS_FOR_TESTING,
         )
         .unwrap();
 
@@ -2574,6 +2581,7 @@ mod tests {
             false,
             false,
             false,
+            crate::accounts_index::BINS_FOR_TESTING,
         )
         .unwrap();
 
@@ -2683,6 +2691,7 @@ mod tests {
             false,
             false,
             false,
+            crate::accounts_index::BINS_FOR_TESTING,
         )
         .unwrap();
 
@@ -2781,6 +2790,7 @@ mod tests {
             false,
             false,
             false,
+            crate::accounts_index::BINS_FOR_TESTING,
         )
         .unwrap();