Add lock to make sure slot-based locktree calls are safe (#7993)

core/src/blockstream_service.rs

@@ -58,31 +58,35 @@ impl BlockstreamService {
         let timeout = Duration::new(1, 0);
         let (slot, slot_leader) = slot_full_receiver.recv_timeout(timeout)?;
 
-        let entries = blockstore.get_slot_entries(slot, 0, None).unwrap();
-        let blockstore_meta = blockstore.meta(slot).unwrap().unwrap();
-        let _parent_slot = if slot == 0 {
-            None
-        } else {
-            Some(blockstore_meta.parent_slot)
-        };
-        let ticks_per_slot = entries.iter().filter(|entry| entry.is_tick()).count() as u64;
-        let mut tick_height = ticks_per_slot * slot;
+        // Slot might not exist due to LedgerCleanupService, check first
+        let blockstore_meta = blockstore.meta(slot).unwrap();
+        if let Some(blockstore_meta) = blockstore_meta {
+            // Return error to main loop. Thread won't exit, will just log the error
+            let entries = blockstore.get_slot_entries(slot, 0, None)?;
+            let _parent_slot = if slot == 0 {
+                None
+            } else {
+                Some(blockstore_meta.parent_slot)
+            };
+            let ticks_per_slot = entries.iter().filter(|entry| entry.is_tick()).count() as u64;
+            let mut tick_height = ticks_per_slot * slot;
 
-        for (i, entry) in entries.iter().enumerate() {
-            if entry.is_tick() {
-                tick_height += 1;
-            }
-            blockstream
-                .emit_entry_event(slot, tick_height, &slot_leader, &entry)
-                .unwrap_or_else(|e| {
-                    debug!("Blockstream error: {:?}, {:?}", e, blockstream.output);
-                });
-            if i == entries.len() - 1 {
+            for (i, entry) in entries.iter().enumerate() {
+                if entry.is_tick() {
+                    tick_height += 1;
+                }
                 blockstream
-                    .emit_block_event(slot, tick_height, &slot_leader, entry.hash)
+                    .emit_entry_event(slot, tick_height, &slot_leader, &entry)
                     .unwrap_or_else(|e| {
                         debug!("Blockstream error: {:?}, {:?}", e, blockstream.output);
                     });
+                if i == entries.len() - 1 {
+                    blockstream
+                        .emit_block_event(slot, tick_height, &slot_leader, entry.hash)
+                        .unwrap_or_else(|e| {
+                            debug!("Blockstream error: {:?}, {:?}", e, blockstream.output);
+                        });
+                }
             }
         }
         Ok(())

core/src/ledger_cleanup_service.rs

@@ -68,9 +68,13 @@ impl LedgerCleanupService {
         let disk_utilization_pre = blockstore.storage_size();
 
         let root = new_root_receiver.recv_timeout(Duration::from_secs(1))?;
+
+        // Notify blockstore of impending purge
         if root > *next_purge_batch {
             //cleanup
-            blockstore.purge_slots(0, Some(root - max_ledger_slots));
+            let lowest_slot = root - max_ledger_slots;
+            *blockstore.lowest_cleanup_slot.write().unwrap() = lowest_slot;
+            blockstore.purge_slots(0, Some(lowest_slot));
             *next_purge_batch += DEFAULT_PURGE_BATCH_SIZE;
         }
 

core/src/replay_stage.rs

@@ -527,6 +527,9 @@ impl ReplayStage {
         let tx_count = tx_count_after - tx_count_before;
 
         confirm_result.map_err(|err| {
+            // LedgerCleanupService should not be cleaning up anything
+            // that comes after the root, so we should not see any
+            // errors related to the slot being purged
             let slot = bank.slot();
             warn!("Fatal replay error in slot: {}, err: {:?}", slot, err);
             datapoint_error!(

core/src/rpc.rs

@@ -383,7 +383,11 @@ impl JsonRpcRequestProcessor {
         let stakes = HashMap::new();
         let stakes = bank.epoch_vote_accounts(epoch).unwrap_or(&stakes);
 
-        Ok(self.blockstore.get_block_time(slot, slot_duration, stakes))
+        Ok(self
+            .blockstore
+            .get_block_time(slot, slot_duration, stakes)
+            .ok()
+            .unwrap_or(None))
     }
 }