Account for possibility of cache flush in load() (#15454)

* Account for possibility of cache flush in load()

* More cleaning

* More cleaning

* Remove unused method and some comment cleaning

* Fix typo

* Make the detected impossible purge race panic()!

* Finally revert to original .expect()

* Fix typos...

* Add assertion for max_root for easier reasoning

* Reframe races with LoadHint as possible opt.

* Fix test

* Make race bug tests run longer for less flaky

* Delay the clone-in-lock slow path even for RPC

* Make get_account panic-free & add its onchain ver.

* Fix rebase conflicts...

* Clean up

* Clean up comment

* Revert fn name change

* Fix flaky test...

* fmt...

Co-authored-by: Ryo Onodera <ryoqun@gmail.com>

runtime/src/accounts_index.rs

@@ -554,7 +554,8 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
         bank 5's parent reference keeps bank 4 alive, which will prevent the `Bank::drop()` from
         running and cleaning up bank 4. Furthermore, no cleans can happen past the saved max_root == 1,
         so a potential newer max root at 3 will not clean up any of the ancestors > 1, so slot 4
-        will not be cleaned in the middle of the scan either.
+        will not be cleaned in the middle of the scan either. (NOTE similar reasoning is employed for
+        assert!() justification in AccountsDb::retry_to_get_account_accessor)
         */
         match scan_type {
             ScanTypes::Unindexed(range) => {
@@ -854,21 +855,23 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
     where
         C: Contains<'a, Slot>,
     {
-        let res = {
+        let is_empty = {
             let mut write_account_map_entry = self.get_account_write_entry(pubkey).unwrap();
             write_account_map_entry.slot_list_mut(|slot_list| {
                 slot_list.retain(|(slot, item)| {
                     let should_purge = slots_to_purge.contains(&slot);
                     if should_purge {
                         reclaims.push((*slot, item.clone()));
+                        false
+                    } else {
+                        true
                     }
-                    !should_purge
                 });
                 slot_list.is_empty()
             })
         };
         self.purge_secondary_indexes_by_inner_key(pubkey, Some(slots_to_purge), account_indexes);
-        res
+        is_empty
     }
 
     pub fn min_ongoing_scan_root(&self) -> Option<Slot> {
@@ -1129,14 +1132,15 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
             if should_purge {
                 reclaims.push((*slot, value.clone()));
                 purged_slots.insert(*slot);
+                false
+            } else {
+                true
             }
-            !should_purge
         });
 
         self.purge_secondary_indexes_by_inner_key(pubkey, Some(&purged_slots), account_indexes);
     }
 
-    // `is_cached` closure is needed to work around the generic (`T`) indexed type.
     pub fn clean_rooted_entries(
         &self,
         pubkey: &Pubkey,