gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add ability to prune ledger (#5128)

Browse Source 
* Add utility to prune the ledger

* Add tests

* Fix clippy

* Fix off by one

* Rework to force delete every column

* Minor fixup
This commit is contained in:
Sagar Dhawan
2019-07-17 14:42:29 -07:00
committed by GitHub
parent 027ebb6670
commit 6ad9dc18d8
5 changed files with 179 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@@ -2684,6 +2684,7 @@ name = "solana-ledger-tool"
version = "0.17.0"
dependencies = [
"assert_cmd 0.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
"bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)",

166
core/src/blocktree.rs
View File

@@ -193,6 +193,70 @@ impl Blocktree {
false
}
// silently deletes all blocktree column families starting at the given slot
fn delete_all_columns(&self, starting_slot: u64) {
match self.meta_cf.force_delete_all(Some(starting_slot)) {
Ok(_) => (),
Err(e) => error!(
"Error: {:?} while deleting meta_cf for slot {:?}",
e, starting_slot
),
}
match self.data_cf.force_delete_all(Some((starting_slot, 0))) {
Ok(_) => (),
Err(e) => error!(
"Error: {:?} while deleting data_cf for slot {:?}",
e, starting_slot
),
}
match self
.erasure_meta_cf
.force_delete_all(Some((starting_slot, 0)))
{
Ok(_) => (),
Err(e) => error!(
"Error: {:?} while deleting erasure_meta_cf for slot {:?}",
e, starting_slot
),
}
match self.erasure_cf.force_delete_all(Some((starting_slot, 0))) {
Ok(_) => (),
Err(e) => error!(
"Error: {:?} while deleting erasure_cf for slot {:?}",
e, starting_slot
),
}
match self.orphans_cf.force_delete_all(Some(starting_slot)) {
Ok(_) => (),
Err(e) => error!(
"Error: {:?} while deleting orphans_cf for slot {:?}",
e, starting_slot
),
}
match self.index_cf.force_delete_all(Some(starting_slot)) {
Ok(_) => (),
Err(e) => error!(
"Error: {:?} while deleting index_cf for slot {:?}",
e, starting_slot
),
}
match self.dead_slots_cf.force_delete_all(Some(starting_slot)) {
Ok(_) => (),
Err(e) => error!(
"Error: {:?} while deleting dead_slots_cf for slot {:?}",
e, starting_slot
),
}
let roots_cf = self.db.column::<cf::Root>();
match roots_cf.force_delete_all(Some(starting_slot)) {
Ok(_) => (),
Err(e) => error!(
"Error: {:?} while deleting roots_cf for slot {:?}",
e, starting_slot
),
}
}
pub fn erasure_meta(&self, slot: u64, set_index: u64) -> Result<Option<ErasureMeta>> {
self.erasure_meta_cf.get((slot, set_index))
}
@@ -201,7 +265,7 @@ impl Blocktree {
self.orphans_cf.get(slot)
}
pub fn rooted_slot_iterator<'a>(&'a self, slot: u64) -> Result<RootedSlotIterator<'a>> {
pub fn rooted_slot_iterator(&self, slot: u64) -> Result<RootedSlotIterator> {
RootedSlotIterator::new(slot, self)
}
@@ -512,6 +576,13 @@ impl Blocktree {
self.data_cf.get_bytes((slot, index))
}
/// Manually update the meta for a slot.
/// Can interfere with automatic meta update and potentially break chaining.
/// Dangerous. Use with care.
pub fn put_meta_bytes(&self, slot: u64, bytes: &[u8]) -> Result<()> {
self.meta_cf.put_bytes(slot, bytes)
}
/// For benchmarks, testing, and setup.
/// Does no metadata tracking. Use with care.
pub fn put_data_blob_bytes(&self, slot: u64, index: u64, bytes: &[u8]) -> Result<()> {
@@ -921,6 +992,39 @@ impl Blocktree {
batch_processor.write(batch)?;
Ok(())
}
/// Prune blocktree such that slots higher than `target_slot` are deleted and all references to
/// higher slots are removed
pub fn prune(&self, target_slot: u64) {
let mut meta = self
.meta(target_slot)
.expect("couldn't read slot meta")
.expect("no meta for target slot");
meta.next_slots.clear();
self.put_meta_bytes(
target_slot,
&bincode::serialize(&meta).expect("couldn't get meta bytes"),
)
.expect("unable to update meta for target slot");
self.delete_all_columns(target_slot + 1);
// fixup anything that refers to non-root slots and delete the rest
for (slot, mut meta) in self
.slot_meta_iterator(0)
.expect("unable to iterate over meta")
{
if slot > target_slot {
break;
}
meta.next_slots.retain(|slot| *slot <= target_slot);
self.put_meta_bytes(
slot,
&bincode::serialize(&meta).expect("couldn't update meta"),
)
.expect("couldn't update meta");
}
}
}
fn insert_data_blob_batch<'a, I>(
@@ -3336,6 +3440,66 @@ pub mod tests {
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
}
#[test]
fn test_prune() {
let blocktree_path = get_tmp_ledger_path!();
let blocktree = Blocktree::open(&blocktree_path).unwrap();
let (blobs, _) = make_many_slot_entries(0, 50, 6);
blocktree.write_blobs(blobs).unwrap();
blocktree
.slot_meta_iterator(0)
.unwrap()
.for_each(|(_, meta)| assert_eq!(meta.last_index, 5));
blocktree.prune(5);
blocktree
.slot_meta_iterator(0)
.unwrap()
.for_each(|(slot, meta)| {
assert!(slot <= 5);
assert_eq!(meta.last_index, 5)
});
let data_iter = blocktree.data_cf.iter(Some((0, 0))).unwrap();
for ((slot, _), _) in data_iter {
if slot > 5 {
assert!(false);
}
}
drop(blocktree);
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
}
#[should_panic]
#[test]
fn test_prune_out_of_bounds() {
let blocktree_path = get_tmp_ledger_path!();
let blocktree = Blocktree::open(&blocktree_path).unwrap();
// slot 5 does not exist, prune should panic
blocktree.prune(5);
drop(blocktree);
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
}
#[test]
fn test_iter_bounds() {
let blocktree_path = get_tmp_ledger_path!();
let blocktree = Blocktree::open(&blocktree_path).unwrap();
// slot 5 does not exist, iter should be ok and should be a noop
blocktree
.slot_meta_iterator(5)
.unwrap()
.for_each(|_| assert!(false));
drop(blocktree);
Blocktree::destroy(&blocktree_path).expect("Expected successful database destruction");
}
mod erasure {
use super::*;
use crate::blocktree::meta::ErasureMetaStatus;

10
core/src/blocktree/db.rs
View File

@@ -409,6 +409,16 @@ where
Ok(iter.map(|(key, value)| (C::index(&key), value)))
}
//TODO add a delete_until that goes the other way
pub fn force_delete_all(&self, start_from: Option<C::Index>) -> Result<()> {
let iter = self.iter(start_from)?;
iter.for_each(|(index, _)| match self.delete(index) {
Ok(_) => (),
Err(e) => error!("Error: {:?} while deleting {:?}", e, C::NAME),
});
Ok(())
}
#[inline]
pub fn handle(&self) -> B::ColumnFamily {
self.backend.cf_handle(C::NAME).clone()

1
ledger-tool/Cargo.toml
View File

@@ -9,6 +9,7 @@ license = "Apache-2.0"
homepage = "https://solana.com/"
[dependencies]
bincode = "1.1.4"
clap = "2.33.0"
serde = "1.0.94"
serde_derive = "1.0.94"

3
ledger-tool/src/main.rs
View File

@@ -91,6 +91,7 @@ fn main() {
.long("slot-list")
.value_name("FILENAME")
.takes_value(true)
.required(true)
.help("The location of the YAML file with a list of rollback slot heights and hashes"),
))
.subcommand(SubCommand::with_name("list-roots").about("Output upto last <num-roots> root hashes and their heights starting at the given block height").arg(
@@ -191,7 +192,7 @@ fn main() {
.last()
.expect("Failed to find a valid slot");
println!("Prune at slot {:?} hash {:?}", target_slot, target_hash);
// ToDo: Do the actual pruning of the database
blocktree.prune(*target_slot);
}
}
("list-roots", Some(args_matches)) => {