Explicitly sanitize program id indexes before usage

1. check transaction has valid program_id before using it to avoid possible panic;
2. change calculate_cost function signature to return Result;
3. add CostModelError enum, update return type from Result<_, str> to Result<_, CostModelError>

ledger-tool/src/main.rs

@@ -751,7 +751,16 @@ fn compute_slot_cost(blockstore: &Blockstore, slot: Slot) -> Result<(), String>
         let mut cost_model = cost_model.write().unwrap();
         for transaction in &entry.transactions {
             programs += transaction.message().instructions.len();
-            let tx_cost = cost_model.calculate_cost(transaction);
+            let tx_cost = match cost_model.calculate_cost(transaction) {
+                Err(err) => {
+                    warn!(
+                        "failed to calculate transaction cost, err {:?}, tx {:?}",
+                        err, transaction
+                    );
+                    continue;
+                }
+                Ok(cost) => cost,
+            };
             if cost_tracker.try_add(tx_cost).is_err() {
                 println!(
                     "Slot: {}, CostModel rejected transaction {:?}, stats {:?}!",