From e8040a828d7c266972cdab11c840e0aa54eb165c Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Sat, 7 Aug 2021 06:16:43 +0000 Subject: [PATCH] Clean within shrink_all_slots (#19042) (#19109) Co-authored-by: Carl Lin (cherry picked from commit 592013eaf45051581151e5175d0083f13e2a8163) Co-authored-by: sakridge --- runtime/src/accounts_db.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index d9e712868e..9f308f0139 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -2472,12 +2472,20 @@ impl AccountsDb { } pub fn shrink_all_slots(&self, is_startup: bool) { + const DIRTY_STORES_CLEANING_THRESHOLD: usize = 10_000; + const OUTER_CHUNK_SIZE: usize = 2000; + const INNER_CHUNK_SIZE: usize = OUTER_CHUNK_SIZE / 8; if is_startup && self.caching_enabled { let slots = self.all_slots_in_storage(); - let chunk_size = std::cmp::max(slots.len() / 8, 1); // approximately 400k slots in a snapshot - slots.par_chunks(chunk_size).for_each(|slots| { - for slot in slots { - self.shrink_slot_forced(*slot, is_startup); + let inner_chunk_size = std::cmp::max(INNER_CHUNK_SIZE, 1); + slots.chunks(OUTER_CHUNK_SIZE).for_each(|chunk| { + chunk.par_chunks(inner_chunk_size).for_each(|slots| { + for slot in slots { + self.shrink_slot_forced(*slot, is_startup); + } + }); + if self.dirty_stores.len() > DIRTY_STORES_CLEANING_THRESHOLD { + self.clean_accounts(None, is_startup); } }); } else { @@ -2487,6 +2495,9 @@ impl AccountsDb { } else { self.do_shrink_slot_forced_v1(slot); } + if self.dirty_stores.len() > DIRTY_STORES_CLEANING_THRESHOLD { + self.clean_accounts(None, is_startup); + } } } }