Accounts cleanup service and perf improvements (#8799)

* Use atomic for ref count instead of taking rwlock

* Accounts cleanup service

* Review comments

core/src/accounts_cleanup_service.rs

@@ -0,0 +1,38 @@
+// Service to clean up dead slots in accounts_db
+//
+// This can be expensive since we have to walk the append vecs being cleaned up.
+
+use solana_ledger::bank_forks::BankForks;
+use std::sync::{
+    atomic::{AtomicBool, Ordering},
+    Arc, RwLock,
+};
+use std::thread::{self, sleep, Builder, JoinHandle};
+use std::time::Duration;
+
+pub struct AccountsCleanupService {
+    t_cleanup: JoinHandle<()>,
+}
+
+impl AccountsCleanupService {
+    pub fn new(bank_forks: Arc<RwLock<BankForks>>, exit: &Arc<AtomicBool>) -> Self {
+        info!("AccountsCleanupService active");
+        let exit = exit.clone();
+        let t_cleanup = Builder::new()
+            .name("solana-accounts-cleanup".to_string())
+            .spawn(move || loop {
+                if exit.load(Ordering::Relaxed) {
+                    break;
+                }
+                let bank = bank_forks.read().unwrap().working_bank();
+                bank.clean_dead_slots();
+                sleep(Duration::from_millis(100));
+            })
+            .unwrap();
+        Self { t_cleanup }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.t_cleanup.join()
+    }
+}

core/src/lib.rs

@@ -5,6 +5,7 @@
 //! command-line tools to spin up validators and a Rust library
 //!
 
+pub mod accounts_cleanup_service;
 pub mod accounts_hash_verifier;
 pub mod banking_stage;
 pub mod broadcast_stage;

core/src/tvu.rs

@@ -2,6 +2,7 @@
 //! validation pipeline in software.
 
 use crate::{
+    accounts_cleanup_service::AccountsCleanupService,
     accounts_hash_verifier::AccountsHashVerifier,
     broadcast_stage::RetransmitSlotsSender,
     cluster_info::ClusterInfo,
@@ -47,6 +48,7 @@ pub struct Tvu {
     retransmit_stage: RetransmitStage,
     replay_stage: ReplayStage,
     ledger_cleanup_service: Option<LedgerCleanupService>,
+    accounts_cleanup_service: AccountsCleanupService,
     storage_stage: StorageStage,
     accounts_hash_verifier: AccountsHashVerifier,
 }
@@ -205,6 +207,8 @@ impl Tvu {
             )
         });
 
+        let accounts_cleanup_service = AccountsCleanupService::new(bank_forks.clone(), &exit);
+
         let storage_stage = StorageStage::new(
             storage_state,
             root_bank_receiver,
@@ -222,6 +226,7 @@ impl Tvu {
             retransmit_stage,
             replay_stage,
             ledger_cleanup_service,
+            accounts_cleanup_service,
             storage_stage,
             accounts_hash_verifier,
         }
@@ -235,6 +240,7 @@ impl Tvu {
         if self.ledger_cleanup_service.is_some() {
             self.ledger_cleanup_service.unwrap().join()?;
         }
+        self.accounts_cleanup_service.join()?;
         self.replay_stage.join()?;
         self.accounts_hash_verifier.join()?;
         Ok(())