Persist cost table to blockstore (#18123)

* Add `ProgramCosts` Column Family to blockstore, implement LedgerColumn; add `delete_cf` to Rocks
* Add ProgramCosts to compaction excluding list alone side with TransactionStatusIndex in one place: `excludes_from_compaction()`

* Write cost table to blockstore after `replay_stage` replayed active banks; add stats to measure persist time
* Deletes program from `ProgramCosts` in blockstore when they are removed from cost_table in memory
* Only try to persist to blockstore when cost_table is changed.
* Restore cost table during validator startup

* Offload `cost_model` related operations from replay main thread to dedicated service thread, add channel to send execute_timings between these threads;
* Move `cost_update_service` to its own module; replay_stage is now decoupled from cost_model.

core/src/validator.rs

@@ -662,6 +662,7 @@ impl Validator {
             ACCOUNT_MAX_COST,
             BLOCK_MAX_COST,
         )));
+        Self::initiate_cost_model(&cost_model, &blockstore.read_program_costs().unwrap());
 
         let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
         let (verified_vote_sender, verified_vote_receiver) = unbounded();
@@ -892,6 +893,31 @@ impl Validator {
             ip_echo_server.shutdown_background();
         }
     }
+
+    fn initiate_cost_model(cost_model: &RwLock<CostModel>, cost_table: &[(Pubkey, u64)]) {
+        let mut cost_model_mutable = cost_model.write().unwrap();
+        for (program_id, cost) in cost_table {
+            match cost_model_mutable.upsert_instruction_cost(program_id, cost) {
+                Ok(c) => {
+                    debug!(
+                        "initiating cost table, instruction {:?} has cost {}",
+                        program_id, c
+                    );
+                }
+                Err(err) => {
+                    debug!(
+                        "initiating cost table, failed for instruction {:?}, err: {}",
+                        program_id, err
+                    );
+                }
+            }
+        }
+        drop(cost_model_mutable);
+        debug!(
+            "restored cost model instruction cost table from blockstore, current values: {:?}",
+            cost_model.read().unwrap().get_instruction_cost_table()
+        );
+    }
 }
 
 fn active_vote_account_exists_in_bank(bank: &Arc<Bank>, vote_account: &Pubkey) -> bool {