investigate system performance test degradation (#17919)

* Add stats and counter around cost model ops, mainly:
- calculate transaction cost
- check transaction can fit in a block
- update block cost tracker after transactions are added to block
- replay_stage to update/insert execution cost to table

* Change mutex on cost_tracker to RwLock

* removed cloning cost_tracker for local use, as the metrics show clone is very expensive.

* acquire and hold locks for block of TXs, instead of acquire and release per transaction;

* remove redundant would_fit check from cost_tracker update execution path

* refactor cost checking with less frequent lock acquiring

* avoid many Transaction_cost heap allocation when calculate cost, which
is in the hot path - executed per transaction.

* create hashmap with new_capacity to reduce runtime heap realloc.

* code review changes: categorize stats, replace explicit drop calls, concisely initiate to default

* address potential deadlock by acquiring locks one at time

core/src/cost_model.rs

@@ -23,6 +23,8 @@ const NON_SIGNED_READONLY_ACCOUNT_ACCESS_COST: u64 = 7;
 pub const ACCOUNT_MAX_COST: u64 = 100_000_000;
 pub const BLOCK_MAX_COST: u64 = 2_500_000_000;
 
+const DEMOTE_SYSVAR_WRITE_LOCKS: bool = true;
+
 // cost of transaction is made of account_access_cost and instruction execution_cost
 // where
 // account_access_cost is the sum of read/write/sign all accounts included in the transaction
@@ -36,6 +38,21 @@ pub struct TransactionCost {
     pub execution_cost: u64,
 }
 
+impl TransactionCost {
+    pub fn new_with_capacity(capacity: usize) -> Self {
+        Self {
+            writable_accounts: Vec::with_capacity(capacity),
+            ..Self::default()
+        }
+    }
+
+    pub fn reset(&mut self) {
+        self.writable_accounts.clear();
+        self.account_access_cost = 0;
+        self.execution_cost = 0;
+    }
+}
+
 #[derive(Debug)]
 pub struct CostModel {
     account_cost_limit: u64,
@@ -90,6 +107,34 @@ impl CostModel {
         cost
     }
 
+    // calculate `transaction` cost, the result is passed back to caller via mutable
+    // parameter `cost`. Existing content in `cost` will be erased before adding new content
+    // This is to allow this function to reuse pre-allocated memory, as this function
+    // is often on hot-path.
+    pub fn calculate_cost_no_alloc(&self, transaction: &Transaction, cost: &mut TransactionCost) {
+        cost.reset();
+
+        let message = transaction.message();
+        message.account_keys.iter().enumerate().for_each(|(i, k)| {
+            let is_signer = message.is_signer(i);
+            let is_writable = message.is_writable(i, DEMOTE_SYSVAR_WRITE_LOCKS);
+
+            if is_signer && is_writable {
+                cost.writable_accounts.push(*k);
+                cost.account_access_cost += SIGNED_WRITABLE_ACCOUNT_ACCESS_COST;
+            } else if is_signer && !is_writable {
+                cost.account_access_cost += SIGNED_READONLY_ACCOUNT_ACCESS_COST;
+            } else if !is_signer && is_writable {
+                cost.writable_accounts.push(*k);
+                cost.account_access_cost += NON_SIGNED_WRITABLE_ACCOUNT_ACCESS_COST;
+            } else {
+                cost.account_access_cost += NON_SIGNED_READONLY_ACCOUNT_ACCESS_COST;
+            }
+        });
+        cost.execution_cost = self.find_transaction_cost(transaction);
+        debug!("transaction {:?} has cost {:?}", transaction, cost);
+    }
+
     // To update or insert instruction cost to table.
     pub fn upsert_instruction_cost(
         &mut self,
@@ -400,6 +445,33 @@ mod tests {
         assert_eq!(2, tx_cost.writable_accounts.len());
     }
 
+    #[test]
+    fn test_cost_model_calculate_cost_no_alloc() {
+        let (mint_keypair, start_hash) = test_setup();
+        let tx =
+            system_transaction::transfer(&mint_keypair, &Keypair::new().pubkey(), 2, start_hash);
+
+        let expected_account_cost = SIGNED_WRITABLE_ACCOUNT_ACCESS_COST
+            + NON_SIGNED_WRITABLE_ACCOUNT_ACCESS_COST
+            + NON_SIGNED_READONLY_ACCOUNT_ACCESS_COST;
+        let expected_execution_cost = 8;
+
+        let mut cost_model = CostModel::default();
+        cost_model
+            .upsert_instruction_cost(&system_program::id(), &expected_execution_cost)
+            .unwrap();
+
+        // allocate cost, set some random number
+        let mut tx_cost = TransactionCost::new_with_capacity(8);
+        tx_cost.execution_cost = 101;
+        tx_cost.writable_accounts.push(Pubkey::new_unique());
+
+        cost_model.calculate_cost_no_alloc(&tx, &mut tx_cost);
+        assert_eq!(expected_account_cost, tx_cost.account_access_cost);
+        assert_eq!(expected_execution_cost, tx_cost.execution_cost);
+        assert_eq!(2, tx_cost.writable_accounts.len());
+    }
+
     #[test]
     fn test_cost_model_update_instruction_cost() {
         let key1 = Pubkey::new_unique();