calculate hash from store instead of index (#15034)

* calculate hash from store instead of index

* restore update hash in abs

accounts-bench/src/main.rs

@@ -1,3 +1,5 @@
+#[macro_use]
+extern crate log;
 use clap::{crate_description, crate_name, value_t, App, Arg};
 use rayon::prelude::*;
 use solana_measure::measure::Measure;
@@ -51,6 +53,7 @@ fn main() {
 
     let path = PathBuf::from(env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_owned()))
         .join("accounts-bench");
+    println!("cleaning file system: {:?}", path);
     if fs::remove_dir_all(path.clone()).is_err() {
         println!("Warning: Couldn't remove {:?}", path);
     }
@@ -84,6 +87,8 @@ fn main() {
         ancestors.insert(i as u64, i - 1);
         accounts.add_root(i as u64);
     }
+    let mut elapsed = vec![0; iterations];
+    let mut elapsed_store = vec![0; iterations];
     for x in 0..iterations {
         if clean {
             let mut time = Measure::start("clean");
@@ -97,13 +102,39 @@ fn main() {
         } else {
             let mut pubkeys: Vec<Pubkey> = vec![];
             let mut time = Measure::start("hash");
-            let hash = accounts
+            let results = accounts
                 .accounts_db
-                .update_accounts_hash(0, &ancestors, true)
+                .update_accounts_hash(0, &ancestors, true);
-                .0;
             time.stop();
-            println!("hash: {} {}", hash, time);
+            let mut time_store = Measure::start("hash using store");
-            create_test_accounts(&accounts, &mut pubkeys, 1, 0);
+            let results_store = accounts.accounts_db.update_accounts_hash_with_index_option(
+                true,
+                false,
+                solana_sdk::clock::Slot::default(),
+                &ancestors,
+                true,
+            );
+            time_store.stop();
+            if results != results_store {
+                error!("results different: \n{:?}\n{:?}", results, results_store);
             }
+            println!(
+                "hash,{},{},{},{}%",
+                results.0,
+                time,
+                time_store,
+                (time_store.as_us() as f64 / time.as_us() as f64 * 100.0f64) as u32
+            );
+            create_test_accounts(&accounts, &mut pubkeys, 1, 0);
+            elapsed[x] = time.as_us();
+            elapsed_store[x] = time_store.as_us();
+        }
+    }
+
+    for x in elapsed {
+        info!("update_accounts_hash(us),{}", x);
+    }
+    for x in elapsed_store {
+        info!("calculate_accounts_hash_without_index(us),{}", x);
     }
 }

core/src/accounts_hash_verifier.rs

@@ -8,7 +8,9 @@ use crate::{
     cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES},
     snapshot_packager_service::PendingSnapshotPackage,
 };
-use solana_runtime::snapshot_package::{AccountsPackage, AccountsPackageReceiver};
+use solana_runtime::snapshot_package::{
+    AccountsPackage, AccountsPackagePre, AccountsPackageReceiver,
+};
 use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
 use std::collections::{HashMap, HashSet};
 use std::{
@@ -49,7 +51,7 @@ impl AccountsHashVerifier {
 
                     match accounts_package_receiver.recv_timeout(Duration::from_secs(1)) {
                         Ok(accounts_package) => {
-                            Self::process_accounts_package(
+                            Self::process_accounts_package_pre(
                                 accounts_package,
                                 &cluster_info,
                                 &trusted_validators,
@@ -72,6 +74,32 @@ impl AccountsHashVerifier {
         }
     }
 
+    fn process_accounts_package_pre(
+        accounts_package: AccountsPackagePre,
+        cluster_info: &ClusterInfo,
+        trusted_validators: &Option<HashSet<Pubkey>>,
+        halt_on_trusted_validator_accounts_hash_mismatch: bool,
+        pending_snapshot_package: &Option<PendingSnapshotPackage>,
+        hashes: &mut Vec<(Slot, Hash)>,
+        exit: &Arc<AtomicBool>,
+        fault_injection_rate_slots: u64,
+        snapshot_interval_slots: u64,
+    ) {
+        let accounts_package =
+            solana_runtime::snapshot_utils::process_accounts_package_pre(accounts_package);
+        Self::process_accounts_package(
+            accounts_package,
+            cluster_info,
+            trusted_validators,
+            halt_on_trusted_validator_accounts_hash_mismatch,
+            pending_snapshot_package,
+            hashes,
+            exit,
+            fault_injection_rate_slots,
+            snapshot_interval_slots,
+        );
+    }
+
     fn process_accounts_package(
         accounts_package: AccountsPackage,
         cluster_info: &ClusterInfo,
@@ -83,6 +111,7 @@ impl AccountsHashVerifier {
         fault_injection_rate_slots: u64,
         snapshot_interval_slots: u64,
     ) {
+        let hash = accounts_package.hash;
         if fault_injection_rate_slots != 0
             && accounts_package.slot % fault_injection_rate_slots == 0
         {
@@ -91,10 +120,10 @@ impl AccountsHashVerifier {
             use solana_sdk::hash::extend_and_hash;
             warn!("inserting fault at slot: {}", accounts_package.slot);
             let rand = thread_rng().gen_range(0, 10);
-            let hash = extend_and_hash(&accounts_package.hash, &[rand]);
+            let hash = extend_and_hash(&hash, &[rand]);
             hashes.push((accounts_package.slot, hash));
         } else {
-            hashes.push((accounts_package.slot, accounts_package.hash));
+            hashes.push((accounts_package.slot, hash));
         }
 
         while hashes.len() > MAX_SNAPSHOT_HASHES {

core/src/snapshot_packager_service.rs

@@ -156,7 +156,7 @@ mod tests {
 
         // Create a packageable snapshot
         let output_tar_path = snapshot_utils::get_snapshot_archive_path(
-            &snapshot_package_output_path,
+            snapshot_package_output_path,
             &(42, Hash::default()),
             ArchiveFormat::TarBzip2,
         );

core/src/tvu.rs

@@ -79,6 +79,7 @@ pub struct TvuConfig {
     pub repair_validators: Option<HashSet<Pubkey>>,
     pub accounts_hash_fault_injection_slots: u64,
     pub accounts_db_caching_enabled: bool,
+    pub test_hash_calculation: bool,
 }
 
 impl Tvu {
@@ -274,6 +275,7 @@ impl Tvu {
             &exit,
             accounts_background_request_handler,
             tvu_config.accounts_db_caching_enabled,
+            tvu_config.test_hash_calculation,
         );
 
         Tvu {

core/src/validator.rs

@@ -121,6 +121,7 @@ pub struct ValidatorConfig {
     pub account_indexes: HashSet<AccountIndex>,
     pub accounts_db_caching_enabled: bool,
     pub warp_slot: Option<Slot>,
+    pub accounts_db_test_hash_calculation: bool,
 }
 
 impl Default for ValidatorConfig {
@@ -168,6 +169,7 @@ impl Default for ValidatorConfig {
             account_indexes: HashSet::new(),
             accounts_db_caching_enabled: false,
             warp_slot: None,
+            accounts_db_test_hash_calculation: false,
         }
     }
 }
@@ -641,6 +643,7 @@ impl Validator {
                 repair_validators: config.repair_validators.clone(),
                 accounts_hash_fault_injection_slots: config.accounts_hash_fault_injection_slots,
                 accounts_db_caching_enabled: config.accounts_db_caching_enabled,
+                test_hash_calculation: config.accounts_db_test_hash_calculation,
             },
         );
 

core/tests/snapshots.rs

@@ -153,7 +153,7 @@ mod tests {
                 .unwrap()
                 .snapshot_path,
             snapshot_utils::get_snapshot_archive_path(
-                snapshot_package_output_path,
+                snapshot_package_output_path.to_path_buf(),
                 &(old_last_bank.slot(), old_last_bank.get_accounts_hash()),
                 ArchiveFormat::TarBzip2,
             ),
@@ -218,7 +218,8 @@ mod tests {
             if slot % set_root_interval == 0 || slot == last_slot - 1 {
                 // set_root should send a snapshot request
                 bank_forks.set_root(bank.slot(), &request_sender, None);
-                snapshot_request_handler.handle_snapshot_requests(false);
+                bank.update_accounts_hash();
+                snapshot_request_handler.handle_snapshot_requests(false, false);
             }
         }
 
@@ -238,8 +239,10 @@ mod tests {
             last_bank.get_snapshot_storages(),
             ArchiveFormat::TarBzip2,
             snapshot_version,
+            None,
         )
         .unwrap();
+        let snapshot_package = snapshot_utils::process_accounts_package_pre(snapshot_package);
         snapshot_utils::archive_snapshot_package(&snapshot_package).unwrap();
 
         // Restore bank from snapshot
@@ -358,6 +361,7 @@ mod tests {
                 &snapshot_package_output_path,
                 snapshot_config.snapshot_version,
                 &snapshot_config.archive_format,
+                None,
             )
             .unwrap();
 
@@ -383,7 +387,7 @@ mod tests {
                 fs_extra::dir::copy(&last_snapshot_path, &saved_snapshots_dir, &options).unwrap();
 
                 saved_archive_path = Some(snapshot_utils::get_snapshot_archive_path(
-                    snapshot_package_output_path,
+                    snapshot_package_output_path.to_path_buf(),
                     &(slot, accounts_hash),
                     ArchiveFormat::TarBzip2,
                 ));
@@ -425,6 +429,10 @@ mod tests {
                         snapshot_package = new_snapshot_package;
                     }
 
+                    let snapshot_package =
+                        solana_runtime::snapshot_utils::process_accounts_package_pre(
+                            snapshot_package,
+                        );
                     *pending_snapshot_package.lock().unwrap() = Some(snapshot_package);
                 }
 

download-utils/src/lib.rs

@@ -180,7 +180,7 @@ pub fn download_snapshot(
         ArchiveFormat::TarBzip2,
     ] {
         let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path(
-            ledger_path,
+            ledger_path.to_path_buf(),
             &desired_snapshot_hash,
             *compression,
         );

local-cluster/tests/local_cluster.rs

@@ -1052,7 +1052,10 @@ fn test_snapshot_download() {
 
     trace!("found: {:?}", archive_filename);
     let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
-        &validator_snapshot_test_config.snapshot_output_path,
+        validator_snapshot_test_config
+            .snapshot_output_path
+            .path()
+            .to_path_buf(),
         &archive_snapshot_hash,
         ArchiveFormat::TarBzip2,
     );
@@ -1122,7 +1125,10 @@ fn test_snapshot_restart_tower() {
 
     // Copy archive to validator's snapshot output directory
     let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
-        &validator_snapshot_test_config.snapshot_output_path,
+        validator_snapshot_test_config
+            .snapshot_output_path
+            .path()
+            .to_path_buf(),
         &archive_snapshot_hash,
         ArchiveFormat::TarBzip2,
     );
@@ -1187,7 +1193,10 @@ fn test_snapshots_blockstore_floor() {
 
     // Copy archive to validator's snapshot output directory
     let validator_archive_path = snapshot_utils::get_snapshot_archive_path(
-        &validator_snapshot_test_config.snapshot_output_path,
+        validator_snapshot_test_config
+            .snapshot_output_path
+            .path()
+            .to_path_buf(),
         &(archive_slot, archive_hash),
         ArchiveFormat::TarBzip2,
     );

runtime/Cargo.toml

@@ -14,7 +14,7 @@ blake3 = "0.3.6"
 bv = { version = "0.11.1", features = ["serde"] }
 byteorder = "1.3.4"
 bzip2 = "0.3.3"
-dashmap = { version = "4.0.2", features = ["rayon"] }
+dashmap = { version = "4.0.2", features = ["rayon", "raw-api"] }
 crossbeam-channel = "0.4"
 dir-diff = "0.3.2"
 flate2 = "1.0.14"

runtime/src/accounts_background_service.rs

@@ -77,7 +77,11 @@ pub struct SnapshotRequestHandler {
 
 impl SnapshotRequestHandler {
     // Returns the latest requested snapshot slot, if one exists
-    pub fn handle_snapshot_requests(&self, accounts_db_caching_enabled: bool) -> Option<u64> {
+    pub fn handle_snapshot_requests(
+        &self,
+        accounts_db_caching_enabled: bool,
+        test_hash_calculation: bool,
+    ) -> Option<u64> {
         self.snapshot_request_receiver
             .try_iter()
             .last()
@@ -87,10 +91,6 @@ impl SnapshotRequestHandler {
                     status_cache_slot_deltas,
                 } = snapshot_request;
 
-                let mut hash_time = Measure::start("hash_time");
-                snapshot_root_bank.update_accounts_hash();
-                hash_time.stop();
-
                 let mut shrink_time = Measure::start("shrink_time");
                 if !accounts_db_caching_enabled {
                     snapshot_root_bank
@@ -120,6 +120,15 @@ impl SnapshotRequestHandler {
                 }
                 flush_accounts_cache_time.stop();
 
+                let mut hash_time = Measure::start("hash_time");
+                let mut hash_for_testing = None;
+                snapshot_root_bank
+                    .update_accounts_hash_with_index_option(true, test_hash_calculation);
+                if test_hash_calculation {
+                    hash_for_testing = Some(snapshot_root_bank.get_accounts_hash());
+                }
+                hash_time.stop();
+
                 let mut clean_time = Measure::start("clean_time");
                 // Don't clean the slot we're snapshotting because it may have zero-lamport
                 // accounts that were included in the bank delta hash when the bank was frozen,
@@ -144,6 +153,7 @@ impl SnapshotRequestHandler {
                     &self.snapshot_config.snapshot_package_output_path,
                     self.snapshot_config.snapshot_version,
                     &self.snapshot_config.archive_format,
+                    hash_for_testing,
                 );
                 if r.is_err() {
                     warn!(
@@ -216,11 +226,16 @@ pub struct ABSRequestHandler {
 
 impl ABSRequestHandler {
     // Returns the latest requested snapshot block height, if one exists
-    pub fn handle_snapshot_requests(&self, accounts_db_caching_enabled: bool) -> Option<u64> {
+    pub fn handle_snapshot_requests(
+        &self,
+        accounts_db_caching_enabled: bool,
+        test_hash_calculation: bool,
+    ) -> Option<u64> {
         self.snapshot_request_handler
             .as_ref()
             .and_then(|snapshot_request_handler| {
-                snapshot_request_handler.handle_snapshot_requests(accounts_db_caching_enabled)
+                snapshot_request_handler
+                    .handle_snapshot_requests(accounts_db_caching_enabled, test_hash_calculation)
             })
     }
 
@@ -245,6 +260,7 @@ impl AccountsBackgroundService {
         exit: &Arc<AtomicBool>,
         request_handler: ABSRequestHandler,
         accounts_db_caching_enabled: bool,
+        test_hash_calculation: bool,
     ) -> Self {
         info!("AccountsBackgroundService active");
         let exit = exit.clone();
@@ -287,8 +303,8 @@ impl AccountsBackgroundService {
                 // request for `N` to the snapshot request channel before setting a root `R > N`, and
                 // snapshot_request_handler.handle_requests() will always look for the latest
                 // available snapshot in the channel.
-                let snapshot_block_height =
+                let snapshot_block_height = request_handler
-                    request_handler.handle_snapshot_requests(accounts_db_caching_enabled);
+                    .handle_snapshot_requests(accounts_db_caching_enabled, test_hash_calculation);
                 if accounts_db_caching_enabled {
                     // Note that the flush will do an internal clean of the
                     // cache up to bank.slot(), so should be safe as long

runtime/src/accounts_db.rs

@@ -167,6 +167,27 @@ type ReclaimResult = (AccountSlots, AppendVecOffsets);
 type StorageFinder<'a> = Box<dyn Fn(Slot, usize) -> Arc<AccountStorageEntry> + 'a>;
 type ShrinkCandidates = HashMap<Slot, HashMap<AppendVecId, Arc<AccountStorageEntry>>>;
 
+#[derive(Default, Debug, PartialEq, Clone)]
+struct CalculateHashIntermediate {
+    pub version: u64,
+    pub hash: Hash,
+    pub lamports: u64,
+    pub raw_lamports: u64,
+    pub slot: Slot,
+}
+
+impl CalculateHashIntermediate {
+    pub fn new(version: u64, hash: Hash, lamports: u64, raw_lamports: u64, slot: Slot) -> Self {
+        Self {
+            version,
+            hash,
+            lamports,
+            raw_lamports,
+            slot,
+        }
+    }
+}
+
 trait Versioned {
     fn version(&self) -> u64;
 }
@@ -177,6 +198,12 @@ impl Versioned for (u64, Hash) {
     }
 }
 
+impl Versioned for CalculateHashIntermediate {
+    fn version(&self) -> u64 {
+        self.version
+    }
+}
+
 impl Versioned for (u64, AccountInfo) {
     fn version(&self) -> u64 {
         self.0
@@ -286,6 +313,16 @@ impl<'a> LoadedAccount<'a> {
             },
         }
     }
+
+    pub fn lamports(&self) -> u64 {
+        match self {
+            LoadedAccount::Stored(stored_account_meta) => stored_account_meta.account_meta.lamports,
+            LoadedAccount::Cached((_, cached_account)) => match cached_account {
+                Cow::Owned(cached_account) => cached_account.account.lamports,
+                Cow::Borrowed(cached_account) => cached_account.account.lamports,
+            },
+        }
+    }
 }
 
 #[derive(Clone, Default, Debug)]
@@ -3402,12 +3439,13 @@ impl AccountsDB {
         slot: Slot,
         debug: bool,
     ) -> Hash {
-        let (hash, ..) = Self::accumulate_account_hashes_and_capitalization(hashes, slot, debug).0;
+        let ((hash, ..), ..) =
+            Self::accumulate_account_hashes_and_capitalization(hashes, slot, debug);
         hash
     }
 
     fn sort_hashes_by_pubkey(hashes: &mut Vec<(Pubkey, Hash, u64)>) {
-        hashes.par_sort_by(|a, b| a.0.cmp(&b.0));
+        hashes.par_sort_unstable_by(|a, b| a.0.cmp(&b.0));
     }
 
     fn accumulate_account_hashes_and_capitalization(
@@ -3574,15 +3612,236 @@ impl AccountsDB {
         ancestors: &Ancestors,
         simple_capitalization_enabled: bool,
     ) -> (Hash, u64) {
-        let (hash, total_lamports) = self
+        self.update_accounts_hash_with_index_option(
-            .calculate_accounts_hash(slot, ancestors, false, simple_capitalization_enabled)
+            false,
-            .unwrap();
+            false,
+            slot,
+            ancestors,
+            simple_capitalization_enabled,
+        )
+    }
+
+    pub fn update_accounts_hash_test(
+        &self,
+        slot: Slot,
+        ancestors: &Ancestors,
+        simple_capitalization_enabled: bool,
+    ) -> (Hash, u64) {
+        self.update_accounts_hash_with_index_option(
+            false,
+            true,
+            slot,
+            ancestors,
+            simple_capitalization_enabled,
+        )
+    }
+
+    /// Scan through all the account storage in parallel
+    fn scan_account_storage_no_bank<F, B>(
+        snapshot_storages: SnapshotStorages,
+        scan_func: F,
+    ) -> Vec<B>
+    where
+        F: Fn(LoadedAccount, AppendVecId, &mut B, Slot) + Send + Sync,
+        B: Send + Default,
+    {
+        snapshot_storages
+            .into_par_iter()
+            .flatten()
+            .map(|storage| {
+                let accounts = storage.accounts.accounts(0);
+                let mut retval = B::default();
+                accounts.into_iter().for_each(|stored_account| {
+                    scan_func(
+                        LoadedAccount::Stored(stored_account),
+                        storage.append_vec_id(),
+                        &mut retval,
+                        storage.slot(),
+                    )
+                });
+                retval
+            })
+            .collect()
+    }
+
+    fn remove_zero_balance_accounts(
+        account_maps: DashMap<Pubkey, CalculateHashIntermediate>,
+    ) -> Vec<(Pubkey, Hash, u64)> {
+        type ShardType = dashmap::lock::RwLock<
+            std::collections::HashMap<
+                solana_sdk::pubkey::Pubkey,
+                dashmap::SharedValue<CalculateHashIntermediate>,
+            >,
+        >;
+        let shards: &[ShardType] = account_maps.shards();
+
+        let hashes: Vec<_> = shards
+            .par_iter()
+            .map(|x| {
+                let a: dashmap::lock::RwLockReadGuard<HashMap<_, _>> = x.read();
+                let res: Vec<_> = a
+                    .iter()
+                    .filter_map(|inp| {
+                        let (pubkey, sv) = inp;
+                        let item = sv.get();
+                        if item.raw_lamports != 0 {
+                            Some((*pubkey, item.hash, item.lamports))
+                        } else {
+                            None
+                        }
+                    })
+                    .collect();
+                res
+            })
+            .flatten()
+            .collect();
+        hashes
+    }
+
+    fn rest_of_hash_calculation(
+        accounts: (DashMap<Pubkey, CalculateHashIntermediate>, Measure),
+    ) -> (Hash, u64) {
+        let (account_maps, time_scan) = accounts;
+
+        let mut zeros = Measure::start("eliminate zeros");
+        let hashes = Self::remove_zero_balance_accounts(account_maps);
+        zeros.stop();
+        let hash_total = hashes.len();
+        let (ret, (sort_time, hash_time)) =
+            Self::accumulate_account_hashes_and_capitalization(hashes, Slot::default(), false);
+        datapoint_info!(
+            "calculate_accounts_hash_without_index",
+            ("accounts_scan", time_scan.as_us(), i64),
+            ("eliminate_zeros", zeros.as_us(), i64),
+            ("hash", hash_time.as_us(), i64),
+            ("sort", sort_time.as_us(), i64),
+            ("hash_total", hash_total, i64),
+        );
+
+        ret
+    }
+
+    fn calculate_accounts_hash_helper(
+        &self,
+        do_not_use_index: bool,
+        slot: Slot,
+        ancestors: &Ancestors,
+        simple_capitalization_enabled: bool,
+    ) -> (Hash, u64) {
+        if do_not_use_index {
+            let combined_maps = self.get_snapshot_storages(slot);
+
+            Self::calculate_accounts_hash_without_index(
+                combined_maps,
+                simple_capitalization_enabled,
+            )
+        } else {
+            self.calculate_accounts_hash(slot, ancestors, false, simple_capitalization_enabled)
+                .unwrap()
+        }
+    }
+
+    pub fn update_accounts_hash_with_index_option(
+        &self,
+        do_not_use_index: bool,
+        debug_verify: bool,
+        slot: Slot,
+        ancestors: &Ancestors,
+        simple_capitalization_enabled: bool,
+    ) -> (Hash, u64) {
+        let (hash, total_lamports) = self.calculate_accounts_hash_helper(
+            do_not_use_index,
+            slot,
+            ancestors,
+            simple_capitalization_enabled,
+        );
+        if debug_verify {
+            // calculate the other way (store or non-store) and verify results match.
+            let (hash_other, total_lamports_other) = self.calculate_accounts_hash_helper(
+                !do_not_use_index,
+                slot,
+                ancestors,
+                simple_capitalization_enabled,
+            );
+
+            assert_eq!(hash, hash_other);
+            assert_eq!(total_lamports, total_lamports_other);
+        }
         let mut bank_hashes = self.bank_hashes.write().unwrap();
         let mut bank_hash_info = bank_hashes.get_mut(&slot).unwrap();
         bank_hash_info.snapshot_hash = hash;
         (hash, total_lamports)
     }
 
+    fn handle_one_loaded_account(
+        key: &Pubkey,
+        found_item: CalculateHashIntermediate,
+        map: &DashMap<Pubkey, CalculateHashIntermediate>,
+    ) {
+        match map.entry(*key) {
+            Occupied(mut dest_item) => {
+                let contents = dest_item.get();
+                if contents.slot < found_item.slot
+                    || (contents.slot == found_item.slot
+                        && contents.version() <= found_item.version())
+                {
+                    // replace the item
+                    dest_item.insert(found_item);
+                }
+            }
+            Vacant(v) => {
+                v.insert(found_item);
+            }
+        };
+    }
+
+    fn scan_snapshot_stores(
+        storage: SnapshotStorages,
+        simple_capitalization_enabled: bool,
+    ) -> (DashMap<Pubkey, CalculateHashIntermediate>, Measure) {
+        let map: DashMap<Pubkey, CalculateHashIntermediate> = DashMap::new();
+        let mut time = Measure::start("scan all accounts");
+        Self::scan_account_storage_no_bank(
+            storage,
+            |loaded_account: LoadedAccount,
+             _store_id: AppendVecId,
+             _accum: &mut Vec<(Pubkey, CalculateHashIntermediate)>,
+             slot: Slot| {
+                let version = loaded_account.write_version();
+                let raw_lamports = loaded_account.lamports();
+                let balance = Self::account_balance_for_capitalization(
+                    raw_lamports,
+                    loaded_account.owner(),
+                    loaded_account.executable(),
+                    simple_capitalization_enabled,
+                );
+
+                let source_item = CalculateHashIntermediate::new(
+                    version,
+                    *loaded_account.loaded_hash(),
+                    balance,
+                    raw_lamports,
+                    slot,
+                );
+                Self::handle_one_loaded_account(loaded_account.pubkey(), source_item, &map);
+            },
+        );
+        time.stop();
+
+        (map, time)
+    }
+
+    // modeled after get_accounts_delta_hash
+    // intended to be faster than calculate_accounts_hash
+    pub fn calculate_accounts_hash_without_index(
+        storages: SnapshotStorages,
+        simple_capitalization_enabled: bool,
+    ) -> (Hash, u64) {
+        let result = Self::scan_snapshot_stores(storages, simple_capitalization_enabled);
+
+        Self::rest_of_hash_calculation(result)
+    }
+
     pub fn verify_bank_hash_and_lamports(
         &self,
         slot: Slot,
@@ -4788,6 +5047,163 @@ pub mod tests {
         ancestors
     }
 
+    #[test]
+    fn test_accountsdb_rest_of_hash_calculation() {
+        solana_logger::setup();
+
+        let key = Pubkey::new(&[11u8; 32]);
+        let account_maps: DashMap<Pubkey, CalculateHashIntermediate> = DashMap::new();
+        let hash = Hash::new(&[1u8; 32]);
+        let val = CalculateHashIntermediate::new(0, hash, 88, 490, Slot::default());
+        account_maps.insert(key, val);
+
+        // 2nd key - zero lamports, so will be removed
+        let key = Pubkey::new(&[12u8; 32]);
+        let hash = Hash::new(&[2u8; 32]);
+        let val = CalculateHashIntermediate::new(0, hash, 1, 0, Slot::default());
+        account_maps.insert(key, val);
+
+        let result =
+            AccountsDB::rest_of_hash_calculation((account_maps.clone(), Measure::start("")));
+        let expected_hash = Hash::from_str("8j9ARGFv4W2GfML7d3sVJK2MePwrikqYnu6yqer28cCa").unwrap();
+        assert_eq!(result, (expected_hash, 88));
+
+        // 3rd key - with pubkey value before 1st key so it will be sorted first
+        let key = Pubkey::new(&[10u8; 32]);
+        let hash = Hash::new(&[2u8; 32]);
+        let val = CalculateHashIntermediate::new(0, hash, 20, 20, Slot::default());
+        account_maps.insert(key, val);
+
+        let result = AccountsDB::rest_of_hash_calculation((account_maps, Measure::start("")));
+        let expected_hash = Hash::from_str("EHv9C5vX7xQjjMpsJMzudnDTzoTSRwYkqLzY8tVMihGj").unwrap();
+        assert_eq!(result, (expected_hash, 108));
+    }
+
+    #[test]
+    fn test_accountsdb_handle_one_loaded_account() {
+        solana_logger::setup();
+
+        let account_maps: DashMap<Pubkey, CalculateHashIntermediate> = DashMap::new();
+        let key = Pubkey::new_unique();
+        let hash = Hash::new_unique();
+        let val = CalculateHashIntermediate::new(1, hash, 1, 2, 1);
+
+        AccountsDB::handle_one_loaded_account(&key, val.clone(), &account_maps);
+        assert_eq!(*account_maps.get(&key).unwrap(), val);
+
+        // slot same, version <
+        let hash2 = Hash::new_unique();
+        let val2 = CalculateHashIntermediate::new(0, hash2, 4, 5, 1);
+        AccountsDB::handle_one_loaded_account(&key, val2, &account_maps);
+        assert_eq!(*account_maps.get(&key).unwrap(), val);
+
+        // slot same, vers =
+        let hash3 = Hash::new_unique();
+        let val3 = CalculateHashIntermediate::new(1, hash3, 2, 3, 1);
+        AccountsDB::handle_one_loaded_account(&key, val3.clone(), &account_maps);
+        assert_eq!(*account_maps.get(&key).unwrap(), val3);
+
+        // slot same, vers >
+        let hash4 = Hash::new_unique();
+        let val4 = CalculateHashIntermediate::new(2, hash4, 6, 7, 1);
+        AccountsDB::handle_one_loaded_account(&key, val4.clone(), &account_maps);
+        assert_eq!(*account_maps.get(&key).unwrap(), val4);
+
+        // slot >, version <
+        let hash5 = Hash::new_unique();
+        let val5 = CalculateHashIntermediate::new(0, hash5, 8, 9, 2);
+        AccountsDB::handle_one_loaded_account(&key, val5.clone(), &account_maps);
+        assert_eq!(*account_maps.get(&key).unwrap(), val5);
+    }
+
+    #[test]
+    fn test_accountsdb_remove_zero_balance_accounts() {
+        solana_logger::setup();
+
+        let key = Pubkey::new_unique();
+        let hash = Hash::new_unique();
+        let account_maps: DashMap<Pubkey, CalculateHashIntermediate> = DashMap::new();
+        let val = CalculateHashIntermediate::new(0, hash, 1, 2, Slot::default());
+        account_maps.insert(key, val.clone());
+
+        let result = AccountsDB::remove_zero_balance_accounts(account_maps);
+        assert_eq!(result, vec![(key, val.hash, val.lamports)]);
+
+        // zero original lamports
+        let account_maps: DashMap<Pubkey, CalculateHashIntermediate> = DashMap::new();
+        let val = CalculateHashIntermediate::new(0, hash, 1, 0, Slot::default());
+        account_maps.insert(key, val);
+
+        let result = AccountsDB::remove_zero_balance_accounts(account_maps);
+        assert_eq!(result, vec![]);
+    }
+
+    #[test]
+    fn test_accountsdb_calculate_accounts_hash_without_index_simple() {
+        solana_logger::setup();
+
+        let (storages, _size, _slot_expected) = sample_storage();
+        let result = AccountsDB::calculate_accounts_hash_without_index(storages, true);
+        let expected_hash = Hash::from_str("GKot5hBsd81kMupNCXHaqbhv3huEbxAFMLnpcX2hniwn").unwrap();
+        assert_eq!(result, (expected_hash, 0));
+    }
+
+    fn sample_storage() -> (SnapshotStorages, usize, Slot) {
+        let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
+        let slot_expected: Slot = 0;
+        let size: usize = 123;
+        let data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
+
+        let arc = Arc::new(data);
+        let storages = vec![vec![arc]];
+        (storages, size, slot_expected)
+    }
+
+    #[test]
+    fn test_accountsdb_scan_account_storage_no_bank() {
+        solana_logger::setup();
+
+        let expected = 1;
+        let tf = crate::append_vec::test_utils::get_append_vec_path(
+            "test_accountsdb_scan_account_storage_no_bank",
+        );
+        let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
+        let slot_expected: Slot = 0;
+        let size: usize = 123;
+        let mut data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
+        let av = AppendVec::new(&tf.path, true, 1024 * 1024);
+        data.accounts = av;
+
+        let arc = Arc::new(data);
+        let storages = vec![vec![arc]];
+        let pubkey = solana_sdk::pubkey::new_rand();
+        let acc = Account::new(1, 48, &Account::default().owner);
+        let sm = StoredMeta {
+            data_len: 1,
+            pubkey,
+            write_version: 1,
+        };
+        storages[0][0]
+            .accounts
+            .append_accounts(&[(sm, &acc)], &[Hash::default()]);
+
+        let calls = AtomicU64::new(0);
+        let result = AccountsDB::scan_account_storage_no_bank(
+            storages,
+            |loaded_account: LoadedAccount,
+             _store_id: AppendVecId,
+             accum: &mut Vec<u64>,
+             slot: Slot| {
+                calls.fetch_add(1, Ordering::Relaxed);
+                assert_eq!(loaded_account.pubkey(), &pubkey);
+                assert_eq!(slot_expected, slot);
+                accum.push(expected);
+            },
+        );
+        assert_eq!(calls.load(Ordering::Relaxed), 1);
+        assert_eq!(result, vec![vec![expected]]);
+    }
+
     #[test]
     fn test_accountsdb_compute_merkle_root_and_capitalization() {
         solana_logger::setup();
@@ -4841,10 +5257,18 @@ pub mod tests {
                 } else {
                     result = AccountsDB::accumulate_account_hashes_and_capitalization(
                         input.clone(),
-                        0,
+                        Slot::default(),
                         false,
                     )
                     .0;
+                    assert_eq!(
+                        AccountsDB::accumulate_account_hashes(
+                            input.clone(),
+                            Slot::default(),
+                            false
+                        ),
+                        result.0
+                    );
                     AccountsDB::sort_hashes_by_pubkey(&mut input);
                 }
                 let mut expected = 0;
@@ -6051,12 +6475,12 @@ pub mod tests {
 
         let ancestors = linear_ancestors(current_slot);
         info!("ancestors: {:?}", ancestors);
-        let hash = accounts.update_accounts_hash(current_slot, &ancestors, true);
+        let hash = accounts.update_accounts_hash_test(current_slot, &ancestors, true);
 
         accounts.clean_accounts(None);
 
         assert_eq!(
-            accounts.update_accounts_hash(current_slot, &ancestors, true),
+            accounts.update_accounts_hash_test(current_slot, &ancestors, true),
             hash
         );
 
@@ -6559,7 +6983,7 @@ pub mod tests {
 
         db.store_uncached(some_slot, &[(&key, &account)]);
         db.add_root(some_slot);
-        db.update_accounts_hash(some_slot, &ancestors, true);
+        db.update_accounts_hash_test(some_slot, &ancestors, true);
         assert_matches!(
             db.verify_bank_hash_and_lamports(some_slot, &ancestors, 1, true),
             Ok(_)
@@ -6601,7 +7025,7 @@ pub mod tests {
 
         db.store_uncached(some_slot, &[(&key, &account)]);
         db.add_root(some_slot);
-        db.update_accounts_hash(some_slot, &ancestors, true);
+        db.update_accounts_hash_test(some_slot, &ancestors, true);
         assert_matches!(
             db.verify_bank_hash_and_lamports(some_slot, &ancestors, 1, true),
             Ok(_)
@@ -6615,7 +7039,7 @@ pub mod tests {
                 &solana_sdk::native_loader::create_loadable_account("foo", 1),
             )],
         );
-        db.update_accounts_hash(some_slot, &ancestors, true);
+        db.update_accounts_hash_test(some_slot, &ancestors, true);
         assert_matches!(
             db.verify_bank_hash_and_lamports(some_slot, &ancestors, 1, false),
             Ok(_)
@@ -6644,7 +7068,7 @@ pub mod tests {
             .unwrap()
             .insert(some_slot, BankHashInfo::default());
         db.add_root(some_slot);
-        db.update_accounts_hash(some_slot, &ancestors, true);
+        db.update_accounts_hash_test(some_slot, &ancestors, true);
         assert_matches!(
             db.verify_bank_hash_and_lamports(some_slot, &ancestors, 0, true),
             Ok(_)

runtime/src/bank.rs

@@ -4299,8 +4299,18 @@ impl Bank {
         self.rc.accounts.accounts_db.get_accounts_hash(self.slot)
     }
 
-    pub fn update_accounts_hash(&self) -> Hash {
+    pub fn update_accounts_hash_with_index_option(
-        let (hash, total_lamports) = self.rc.accounts.accounts_db.update_accounts_hash(
+        &self,
+        do_not_use_index: bool,
+        debug_verify: bool,
+    ) -> Hash {
+        let (hash, total_lamports) = self
+            .rc
+            .accounts
+            .accounts_db
+            .update_accounts_hash_with_index_option(
+                do_not_use_index,
+                debug_verify,
                 self.slot(),
                 &self.ancestors,
                 self.simple_capitalization_enabled(),
@@ -4309,6 +4319,10 @@ impl Bank {
         hash
     }
 
+    pub fn update_accounts_hash(&self) -> Hash {
+        self.update_accounts_hash_with_index_option(false, false)
+    }
+
     /// A snapshot bank should be purged of 0 lamport accounts which are not part of the hash
     /// calculation and could shield other real accounts.
     pub fn verify_snapshot_bank(&self) -> bool {
@@ -4942,7 +4956,10 @@ fn is_simple_vote_transaction(transaction: &Transaction) -> bool {
         if program_pubkey == solana_vote_program::id() {
             if let Ok(vote_instruction) = limited_deserialize::<VoteInstruction>(&instruction.data)
             {
-                return matches!(vote_instruction, VoteInstruction::Vote(_) | VoteInstruction::VoteSwitch(_, _));
+                return matches!(
+                    vote_instruction,
+                    VoteInstruction::Vote(_) | VoteInstruction::VoteSwitch(_, _)
+                );
             }
         }
     }
@@ -7955,6 +7972,7 @@ pub(crate) mod tests {
 
     #[test]
     fn test_verify_snapshot_bank() {
+        solana_logger::setup();
         let pubkey = solana_sdk::pubkey::new_rand();
         let (genesis_config, mint_keypair) = create_genesis_config(2_000);
         let bank = Bank::new(&genesis_config);

runtime/src/snapshot_package.rs

@@ -9,11 +9,59 @@ use std::{
 };
 use tempfile::TempDir;
 
-pub type AccountsPackageSender = Sender<AccountsPackage>;
+pub type AccountsPackageSender = Sender<AccountsPackagePre>;
-pub type AccountsPackageReceiver = Receiver<AccountsPackage>;
+pub type AccountsPackageReceiver = Receiver<AccountsPackagePre>;
-pub type AccountsPackageSendError = SendError<AccountsPackage>;
+pub type AccountsPackageSendError = SendError<AccountsPackagePre>;
 
 #[derive(Debug)]
+pub struct AccountsPackagePre {
+    pub slot: Slot,
+    pub block_height: Slot,
+    pub slot_deltas: Vec<BankSlotDelta>,
+    pub snapshot_links: TempDir,
+    pub storages: SnapshotStorages,
+    pub hash: Hash, // temporarily here while we still have to calculate hash before serializing bank
+    pub archive_format: ArchiveFormat,
+    pub snapshot_version: SnapshotVersion,
+    pub snapshot_output_dir: PathBuf,
+    pub expected_capitalization: u64,
+    pub hash_for_testing: Option<Hash>,
+    pub simple_capitalization_testing: bool,
+}
+
+impl AccountsPackagePre {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        slot: Slot,
+        block_height: u64,
+        slot_deltas: Vec<BankSlotDelta>,
+        snapshot_links: TempDir,
+        storages: SnapshotStorages,
+        hash: Hash,
+        archive_format: ArchiveFormat,
+        snapshot_version: SnapshotVersion,
+        snapshot_output_dir: PathBuf,
+        expected_capitalization: u64,
+        hash_for_testing: Option<Hash>,
+        simple_capitalization_testing: bool,
+    ) -> Self {
+        Self {
+            slot,
+            block_height,
+            slot_deltas,
+            snapshot_links,
+            storages,
+            hash,
+            archive_format,
+            snapshot_version,
+            snapshot_output_dir,
+            expected_capitalization,
+            hash_for_testing,
+            simple_capitalization_testing,
+        }
+    }
+}
+
 pub struct AccountsPackage {
     pub slot: Slot,
     pub block_height: Slot,

runtime/src/snapshot_utils.rs

@@ -1,4 +1,5 @@
 use crate::{
+    accounts_db::AccountsDB,
     accounts_index::AccountIndex,
     bank::{Bank, BankSlotDelta, Builtins},
     bank_forks::ArchiveFormat,
@@ -6,7 +7,9 @@ use crate::{
     serde_snapshot::{
         bank_from_stream, bank_to_stream, SerdeStyle, SnapshotStorage, SnapshotStorages,
     },
-    snapshot_package::{AccountsPackage, AccountsPackageSendError, AccountsPackageSender},
+    snapshot_package::{
+        AccountsPackage, AccountsPackagePre, AccountsPackageSendError, AccountsPackageSender,
+    },
 };
 use bincode::{config::Options, serialize_into};
 use bzip2::bufread::BzDecoder;
@@ -144,7 +147,8 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
     snapshot_storages: SnapshotStorages,
     archive_format: ArchiveFormat,
     snapshot_version: SnapshotVersion,
-) -> Result<AccountsPackage> {
+    hash_for_testing: Option<Hash>,
+) -> Result<AccountsPackagePre> {
     // Hard link all the snapshots we need for this package
     let snapshot_tmpdir = tempfile::Builder::new()
         .prefix(&format!("{}{}-", TMP_SNAPSHOT_PREFIX, bank.slot()))
@@ -169,22 +173,19 @@ pub fn package_snapshot<P: AsRef<Path>, Q: AsRef<Path>>(
         )?;
     }
 
-    let snapshot_package_output_file = get_snapshot_archive_path(
+    let package = AccountsPackagePre::new(
-        &snapshot_package_output_path,
-        &(bank.slot(), bank.get_accounts_hash()),
-        archive_format,
-    );
-
-    let package = AccountsPackage::new(
         bank.slot(),
         bank.block_height(),
         status_cache_slot_deltas,
         snapshot_tmpdir,
         snapshot_storages,
-        snapshot_package_output_file,
         bank.get_accounts_hash(),
         archive_format,
         snapshot_version,
+        snapshot_package_output_path.as_ref().to_path_buf(),
+        bank.capitalization(),
+        hash_for_testing,
+        bank.simple_capitalization_enabled(),
     );
 
     Ok(package)
@@ -630,12 +631,12 @@ pub fn bank_from_archive<P: AsRef<Path>>(
     Ok(bank)
 }
 
-pub fn get_snapshot_archive_path<P: AsRef<Path>>(
+pub fn get_snapshot_archive_path(
-    snapshot_output_dir: P,
+    snapshot_output_dir: PathBuf,
     snapshot_hash: &(Slot, Hash),
     archive_format: ArchiveFormat,
 ) -> PathBuf {
-    snapshot_output_dir.as_ref().join(format!(
+    snapshot_output_dir.join(format!(
         "snapshot-{}-{}{}",
         snapshot_hash.0,
         snapshot_hash.1,
@@ -886,6 +887,7 @@ pub fn snapshot_bank(
     snapshot_package_output_path: &Path,
     snapshot_version: SnapshotVersion,
     archive_format: &ArchiveFormat,
+    hash_for_testing: Option<Hash>,
 ) -> Result<()> {
     let storages: Vec<_> = root_bank.get_snapshot_storages();
     let mut add_snapshot_time = Measure::start("add-snapshot-ms");
@@ -908,6 +910,7 @@ pub fn snapshot_bank(
         storages,
         *archive_format,
         snapshot_version,
+        hash_for_testing,
     )?;
 
     accounts_package_sender.send(package)?;
@@ -946,12 +949,55 @@ pub fn bank_to_snapshot_archive<P: AsRef<Path>, Q: AsRef<Path>>(
         storages,
         archive_format,
         snapshot_version,
+        None,
     )?;
 
+    let package = process_accounts_package_pre(package);
+
     archive_snapshot_package(&package)?;
     Ok(package.tar_output_file)
 }
 
+pub fn process_accounts_package_pre(accounts_package: AccountsPackagePre) -> AccountsPackage {
+    let mut time = Measure::start("hash");
+
+    let hash = accounts_package.hash; // temporarily remaining here
+    if let Some(expected_hash) = accounts_package.hash_for_testing {
+        let (hash, lamports) = AccountsDB::calculate_accounts_hash_without_index(
+            accounts_package.storages.clone(),
+            accounts_package.simple_capitalization_testing,
+        );
+        time.stop();
+
+        assert_eq!(accounts_package.expected_capitalization, lamports);
+
+        assert_eq!(expected_hash, hash);
+    };
+
+    datapoint_info!(
+        "accounts_hash_verifier",
+        ("calculate_hash", time.as_us(), i64),
+    );
+
+    let tar_output_file = get_snapshot_archive_path(
+        accounts_package.snapshot_output_dir,
+        &(accounts_package.slot, hash),
+        accounts_package.archive_format,
+    );
+
+    AccountsPackage::new(
+        accounts_package.slot,
+        accounts_package.block_height,
+        accounts_package.slot_deltas,
+        accounts_package.snapshot_links,
+        accounts_package.storages,
+        tar_output_file,
+        hash,
+        accounts_package.archive_format,
+        accounts_package.snapshot_version,
+    )
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

validator/src/main.rs

@@ -1418,6 +1418,11 @@ pub fn main() {
                 .long("no-accounts-db-caching")
                 .help("Disables accounts caching"),
         )
+        .arg(
+            Arg::with_name("accounts_db_test_hash_calculation")
+                .long("accounts-db-test-hash-calculation")
+                .help("Enables testing of hash calculation using stores in AccountsHashVerifier. This has a computational cost."),
+        )
         .arg(
             // legacy nop argument
             Arg::with_name("accounts_db_caching_enabled")
@@ -1623,6 +1628,7 @@ pub fn main() {
             .unwrap_or(poh_service::DEFAULT_PINNED_CPU_CORE),
         account_indexes,
         accounts_db_caching_enabled: !matches.is_present("no_accounts_db_caching"),
+        accounts_db_test_hash_calculation: matches.is_present("accounts_db_test_hash_calculation"),
         ..ValidatorConfig::default()
     };