gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

revert-revert-erasure and erasure fixes (#3833)

Browse Source 
* fix erasure, more tests for full blobs, more metrics

* Revert "Revert "Use Rust erasure library and turn on erasure (#3768)" (#3827)"

This reverts commit 4b8cb72977.
This commit is contained in:
Rob Walker
2019-04-17 15:13:54 -07:00
committed by GitHub
parent e03215c4c0
commit 6bef16a6a1
13 changed files with 707 additions and 789 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
core/src/blocktree/meta.rs
View File

@@ -1,4 +1,3 @@
#[cfg(feature = "erasure")]
use crate::erasure::{NUM_CODING, NUM_DATA};
#[derive(Clone, Debug, Default, Deserialize, Serialize, Eq, PartialEq)]
@@ -59,7 +58,6 @@ impl SlotMeta {
}
}
#[cfg(feature = "erasure")]
#[derive(Clone, Debug, Default, Deserialize, Serialize, Eq, PartialEq)]
/// Erasure coding information
pub struct ErasureMeta {
@@ -71,7 +69,13 @@ pub struct ErasureMeta {
pub coding: u64,
}
#[cfg(feature = "erasure")]
#[derive(Debug, PartialEq)]
pub enum ErasureMetaStatus {
CanRecover,
DataFull,
StillNeed(usize),
}
impl ErasureMeta {
pub fn new(set_index: u64) -> ErasureMeta {
ErasureMeta {
@@ -81,46 +85,71 @@ impl ErasureMeta {
}
}
pub fn can_recover(&self) -> bool {
pub fn status(&self) -> ErasureMetaStatus {
let (data_missing, coding_missing) = (
NUM_DATA - self.data.count_ones() as usize,
NUM_CODING - self.coding.count_ones() as usize,
);
data_missing > 0 && data_missing + coding_missing <= NUM_CODING
if data_missing > 0 && data_missing + coding_missing <= NUM_CODING {
ErasureMetaStatus::CanRecover
} else if data_missing == 0 {
ErasureMetaStatus::DataFull
} else {
ErasureMetaStatus::StillNeed(data_missing + coding_missing - NUM_CODING)
}
}
pub fn is_coding_present(&self, index: u64) -> bool {
let set_index = Self::set_index_for(index);
let position = index - self.start_index();
let start = self.start_index();
let end = start + NUM_CODING as u64;
set_index == self.set_index && self.coding & (1 << position) != 0
if start <= index && index < end {
let position = index - start;
self.coding & (1 << position) != 0
} else {
false
}
}
pub fn set_coding_present(&mut self, index: u64) {
pub fn set_coding_present(&mut self, index: u64, present: bool) {
let set_index = Self::set_index_for(index);
if set_index as u64 == self.set_index {
let position = index - self.start_index();
self.coding |= 1 << position;
if present {
self.coding |= 1 << position;
} else {
self.coding &= !(1 << position);
}
}
}
pub fn is_data_present(&self, index: u64) -> bool {
let set_index = Self::set_index_for(index);
let position = index - self.start_index();
let start = self.start_index();
let end = start + NUM_DATA as u64;
set_index == self.set_index && self.data & (1 << position) != 0
if start <= index && index < end {
let position = index - start;
self.data & (1 << position) != 0
} else {
false
}
}
pub fn set_data_present(&mut self, index: u64) {
pub fn set_data_present(&mut self, index: u64, present: bool) {
let set_index = Self::set_index_for(index);
if set_index as u64 == self.set_index {
let position = index - self.start_index();
self.data |= 1 << position;
if present {
self.data |= 1 << position;
} else {
self.data &= !(1 << position);
}
}
}
@@ -139,7 +168,29 @@ impl ErasureMeta {
}
}
#[cfg(feature = "erasure")]
#[test]
fn test_meta_indexes() {
use rand::{thread_rng, Rng};
let mut rng = thread_rng();
for _ in 0..100 {
let set_index = rng.gen_range(0, 1_000);
let blob_index = (set_index * NUM_DATA as u64) + rng.gen_range(0, 16);
assert_eq!(set_index, ErasureMeta::set_index_for(blob_index));
let e_meta = ErasureMeta::new(set_index);
assert_eq!(e_meta.start_index(), set_index * NUM_DATA as u64);
let (data_end_idx, coding_end_idx) = e_meta.end_indexes();
assert_eq!(data_end_idx, (set_index + 1) * NUM_DATA as u64);
assert_eq!(
coding_end_idx,
set_index * NUM_DATA as u64 + NUM_CODING as u64
);
}
}
#[test]
fn test_meta_coding_present() {
let set_index = 0;
@@ -150,7 +201,7 @@ fn test_meta_coding_present() {
};
for i in 0..NUM_CODING as u64 {
e_meta.set_coding_present(i);
e_meta.set_coding_present(i, true);
assert_eq!(e_meta.is_coding_present(i), true);
}
for i in NUM_CODING as u64..NUM_DATA as u64 {
@@ -160,7 +211,7 @@ fn test_meta_coding_present() {
e_meta.set_index = ErasureMeta::set_index_for((NUM_DATA * 17) as u64);
for i in (NUM_DATA * 17) as u64..((NUM_DATA * 17) + NUM_CODING) as u64 {
e_meta.set_coding_present(i);
e_meta.set_coding_present(i, true);
assert_eq!(e_meta.is_coding_present(i), true);
}
for i in (NUM_DATA * 17 + NUM_CODING) as u64..((NUM_DATA * 17) + NUM_DATA) as u64 {
@@ -168,9 +219,8 @@ fn test_meta_coding_present() {
}
}
#[cfg(feature = "erasure")]
#[test]
fn test_can_recover() {
fn test_erasure_meta_status() {
let set_index = 0;
let mut e_meta = ErasureMeta {
set_index,
@@ -178,36 +228,63 @@ fn test_can_recover() {
coding: 0,
};
assert!(!e_meta.can_recover());
assert_eq!(e_meta.status(), ErasureMetaStatus::StillNeed(NUM_DATA));
e_meta.data = 0b1111_1111_1111_1111;
e_meta.coding = 0x00;
assert!(!e_meta.can_recover());
assert_eq!(e_meta.status(), ErasureMetaStatus::DataFull);
e_meta.coding = 0x0e;
assert_eq!(0x0fu8, 0b0000_1111u8);
assert!(!e_meta.can_recover());
assert_eq!(e_meta.status(), ErasureMetaStatus::DataFull);
e_meta.data = 0b0111_1111_1111_1111;
assert!(e_meta.can_recover());
assert_eq!(e_meta.status(), ErasureMetaStatus::CanRecover);
e_meta.data = 0b0111_1111_1111_1110;
assert!(e_meta.can_recover());
assert_eq!(e_meta.status(), ErasureMetaStatus::CanRecover);
e_meta.data = 0b0111_1111_1011_1110;
assert!(e_meta.can_recover());
assert_eq!(e_meta.status(), ErasureMetaStatus::CanRecover);
e_meta.data = 0b0111_1011_1011_1110;
assert!(!e_meta.can_recover());
assert_eq!(e_meta.status(), ErasureMetaStatus::StillNeed(1));
e_meta.data = 0b0111_1011_1011_1110;
assert!(!e_meta.can_recover());
assert_eq!(e_meta.status(), ErasureMetaStatus::StillNeed(1));
e_meta.coding = 0b0000_1110;
e_meta.data = 0b1111_1111_1111_1100;
assert!(e_meta.can_recover());
assert_eq!(e_meta.status(), ErasureMetaStatus::CanRecover);
e_meta.data = 0b1111_1111_1111_1000;
assert!(e_meta.can_recover());
assert_eq!(e_meta.status(), ErasureMetaStatus::CanRecover);
}
#[test]
fn test_meta_data_present() {
let set_index = 0;
let mut e_meta = ErasureMeta {
set_index,
data: 0,
coding: 0,
};
for i in 0..NUM_DATA as u64 {
e_meta.set_data_present(i, true);
assert_eq!(e_meta.is_data_present(i), true);
}
for i in NUM_DATA as u64..2 * NUM_DATA as u64 {
assert_eq!(e_meta.is_data_present(i), false);
}
e_meta.set_index = ErasureMeta::set_index_for((NUM_DATA * 23) as u64);
for i in (NUM_DATA * 23) as u64..(NUM_DATA * 24) as u64 {
e_meta.set_data_present(i, true);
assert_eq!(e_meta.is_data_present(i), true);
}
for i in (NUM_DATA * 22) as u64..(NUM_DATA * 23) as u64 {
assert_eq!(e_meta.is_data_present(i), false);
}
}