gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove ledger purge batching (#10830)

Browse Source
This commit is contained in:
sakridge
2020-06-29 14:44:35 -07:00
committed by GitHub
parent 59aa299d05
commit 583cec922b
7 changed files with 34 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
ledger/src/blockstore.rs
View File

@ -6155,14 +6155,14 @@ pub mod tests {
.insert_shreds(all_shreds, Some(&leader_schedule_cache), false)
.unwrap();
verify_index_integrity(&blockstore, slot);
blockstore.purge_slots(0, slot);
blockstore.purge_and_compact_slots(0, slot);
// Test inserting just the codes, enough for recovery
blockstore
.insert_shreds(coding_shreds.clone(), Some(&leader_schedule_cache), false)
.unwrap();
verify_index_integrity(&blockstore, slot);
blockstore.purge_slots(0, slot);
blockstore.purge_and_compact_slots(0, slot);
// Test inserting some codes, but not enough for recovery
blockstore
@ -6173,7 +6173,7 @@ pub mod tests {
)
.unwrap();
verify_index_integrity(&blockstore, slot);
blockstore.purge_slots(0, slot);
blockstore.purge_and_compact_slots(0, slot);
// Test inserting just the codes, and some data, enough for recovery
let shreds: Vec<_> = data_shreds[..data_shreds.len() - 1]
@ -6185,7 +6185,7 @@ pub mod tests {
.insert_shreds(shreds, Some(&leader_schedule_cache), false)
.unwrap();
verify_index_integrity(&blockstore, slot);
blockstore.purge_slots(0, slot);
blockstore.purge_and_compact_slots(0, slot);
// Test inserting some codes, and some data, but enough for recovery
let shreds: Vec<_> = data_shreds[..data_shreds.len() / 2 - 1]
@ -6197,7 +6197,7 @@ pub mod tests {
.insert_shreds(shreds, Some(&leader_schedule_cache), false)
.unwrap();
verify_index_integrity(&blockstore, slot);
blockstore.purge_slots(0, slot);
blockstore.purge_and_compact_slots(0, slot);
// Test inserting all shreds in 2 rounds, make sure nothing is lost
let shreds1: Vec<_> = data_shreds[..data_shreds.len() / 2 - 1]
@ -6217,7 +6217,7 @@ pub mod tests {
.insert_shreds(shreds2, Some(&leader_schedule_cache), false)
.unwrap();
verify_index_integrity(&blockstore, slot);
blockstore.purge_slots(0, slot);
blockstore.purge_and_compact_slots(0, slot);
// Test not all, but enough data and coding shreds in 2 rounds to trigger recovery,
// make sure nothing is lost
@ -6242,7 +6242,7 @@ pub mod tests {
.insert_shreds(shreds2, Some(&leader_schedule_cache), false)
.unwrap();
verify_index_integrity(&blockstore, slot);
blockstore.purge_slots(0, slot);
blockstore.purge_and_compact_slots(0, slot);
// Test insert shreds in 2 rounds, but not enough to trigger
// recovery, make sure nothing is lost
@ -6267,7 +6267,7 @@ pub mod tests {
.insert_shreds(shreds2, Some(&leader_schedule_cache), false)
.unwrap();
verify_index_integrity(&blockstore, slot);
blockstore.purge_slots(0, slot);
blockstore.purge_and_compact_slots(0, slot);
}
Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
}

74
ledger/src/blockstore/blockstore_purge.rs
View File

@ -1,5 +1,4 @@
use super::*;
use std::time::Instant;
#[derive(Default)]
pub struct PurgeStats {
@ -12,61 +11,28 @@ impl Blockstore {
/// Dangerous; Use with care:
/// Does not check for integrity and does not update slot metas that refer to deleted slots
/// Modifies multiple column families simultaneously
pub fn purge_slots_with_delay(
&self,
from_slot: Slot,
to_slot: Slot,
delay_between_purges: Option<Duration>,
purge_type: PurgeType,
) {
// if there's no upper bound, split the purge request into batches of 1000 slots
const PURGE_BATCH_SIZE: u64 = 1000;
let mut batch_start = from_slot;
pub fn purge_slots(&self, from_slot: Slot, to_slot: Slot, purge_type: PurgeType) {
let mut purge_stats = PurgeStats::default();
let mut last_datapoint = Instant::now();
let mut datapoint_start = batch_start;
while batch_start < to_slot {
let batch_end = (batch_start + PURGE_BATCH_SIZE).min(to_slot);
let purge_result =
self.run_purge_with_stats(from_slot, to_slot, purge_type, &mut purge_stats);
let purge_result =
self.run_purge_with_stats(batch_start, batch_end, purge_type, &mut purge_stats);
if last_datapoint.elapsed().as_millis() > 1000 {
datapoint_info!(
"blockstore-purge",
("from_slot", datapoint_start as i64, i64),
("to_slot", batch_end as i64, i64),
("delete_range_us", purge_stats.delete_range as i64, i64),
("write_batch_us", purge_stats.write_batch as i64, i64)
);
last_datapoint = Instant::now();
purge_stats = PurgeStats::default();
datapoint_start = batch_end;
}
match purge_result {
Ok(_all_columns_purged) => {
batch_start = batch_end;
if let Some(ref duration) = delay_between_purges {
// Cooperate with other blockstore users
std::thread::sleep(*duration);
}
}
Err(e) => {
error!(
"Error: {:?}; Purge failed in range {:?} to {:?}",
e, batch_start, batch_end
);
break;
}
}
datapoint_info!(
"blockstore-purge",
("from_slot", from_slot as i64, i64),
("to_slot", to_slot as i64, i64),
("delete_range_us", purge_stats.delete_range as i64, i64),
("write_batch_us", purge_stats.write_batch as i64, i64)
);
if let Err(e) = purge_result {
error!(
"Error: {:?}; Purge failed in range {:?} to {:?}",
e, from_slot, to_slot
);
}
}
// TODO: rename purge_slots() to purge_and_compact_slots()
pub fn purge_slots(&self, from_slot: Slot, to_slot: Slot) {
self.purge_slots_with_delay(from_slot, to_slot, None, PurgeType::Exact);
pub fn purge_and_compact_slots(&self, from_slot: Slot, to_slot: Slot) {
self.purge_slots(from_slot, to_slot, PurgeType::Exact);
if let Err(e) = self.compact_storage(from_slot, to_slot) {
// This error is not fatal and indicates an internal error?
error!(
@ -443,11 +409,11 @@ pub mod tests {
let (shreds, _) = make_many_slot_entries(0, 50, 5);
blockstore.insert_shreds(shreds, None, false).unwrap();
blockstore.purge_slots(0, 5);
blockstore.purge_and_compact_slots(0, 5);
test_all_empty_or_min(&blockstore, 6);
blockstore.purge_slots(0, 50);
blockstore.purge_and_compact_slots(0, 50);
// min slot shouldn't matter, blockstore should be empty
test_all_empty_or_min(&blockstore, 100);
@ -471,7 +437,7 @@ pub mod tests {
let (shreds, _) = make_many_slot_entries(0, 5000, 10);
blockstore.insert_shreds(shreds, None, false).unwrap();
blockstore.purge_slots(0, 4999);
blockstore.purge_and_compact_slots(0, 4999);
test_all_empty_or_min(&blockstore, 5000);