Write transaction status and fee into persistent store (#7030)

* Pass blocktree into execute_batch, if persist_transaction_status

* Add validator arg to enable persistent transaction status store

* Pass blocktree into banking_stage, if persist_transaction_status

* Add validator params to bash scripts

* Expose actual transaction statuses outside Bank; add tests

* Fix benches

* Offload transaction status writes to a separate thread

* Enable persistent transaction status along with rpc service

* nudge

* Review comments

core/src/transaction_status_service.rs

@@ -0,0 +1,79 @@
+use crate::result::{Error, Result};
+use solana_client::rpc_request::RpcTransactionStatus;
+use solana_ledger::{blocktree::Blocktree, blocktree_processor::TransactionStatusBatch};
+use solana_runtime::bank::Bank;
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        mpsc::{Receiver, RecvTimeoutError},
+        Arc,
+    },
+    thread::{self, Builder, JoinHandle},
+    time::Duration,
+};
+
+pub struct TransactionStatusService {
+    thread_hdl: JoinHandle<()>,
+}
+
+impl TransactionStatusService {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(
+        write_transaction_status_receiver: Receiver<TransactionStatusBatch>,
+        blocktree: Arc<Blocktree>,
+        exit: &Arc<AtomicBool>,
+    ) -> Self {
+        let exit = exit.clone();
+        let thread_hdl = Builder::new()
+            .name("solana-transaction-status-writer".to_string())
+            .spawn(move || loop {
+                if exit.load(Ordering::Relaxed) {
+                    break;
+                }
+                if let Err(e) = Self::write_transaction_status_batch(
+                    &write_transaction_status_receiver,
+                    &blocktree,
+                ) {
+                    match e {
+                        Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
+                        Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
+                        _ => info!("Error from write_transaction_statuses: {:?}", e),
+                    }
+                }
+            })
+            .unwrap();
+        Self { thread_hdl }
+    }
+
+    fn write_transaction_status_batch(
+        write_transaction_status_receiver: &Receiver<TransactionStatusBatch>,
+        blocktree: &Arc<Blocktree>,
+    ) -> Result<()> {
+        let TransactionStatusBatch {
+            bank,
+            transactions,
+            statuses,
+        } = write_transaction_status_receiver.recv_timeout(Duration::from_secs(1))?;
+
+        let slot = bank.slot();
+        for (transaction, status) in transactions.iter().zip(statuses) {
+            if Bank::can_commit(&status) && !transaction.signatures.is_empty() {
+                let fee_calculator = bank
+                    .get_fee_calculator(&transaction.message().recent_blockhash)
+                    .expect("FeeCalculator must exist");
+                let fee = fee_calculator.calculate_fee(transaction.message());
+                blocktree
+                    .write_transaction_status(
+                        (slot, transaction.signatures[0]),
+                        &RpcTransactionStatus { status, fee },
+                    )
+                    .expect("Expect database write to succeed");
+            }
+        }
+        Ok(())
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.thread_hdl.join()
+    }
+}