gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Erasure refinements, fix generating orders table

Browse Source
This commit is contained in:
Stephen Akridge
2018-05-28 10:25:15 -07:00
committed by Greg Fitzgerald
parent 93b6fceb2f
commit d053f78b74
3 changed files with 153 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

184
src/erasure.rs
View File

@@ -150,20 +150,48 @@ pub fn decode_blocks(data: &mut [&mut [u8]], coding: &[&[u8]], erasures: &[i32])
Ok(())
}
// Generate coding blocks in window from consumed to consumed+NUM_DATA
pub fn generate_coding(
re: &BlobRecycler,
window: &mut Vec<Option<SharedBlob>>,
consumed: usize,
) -> Result<()> {
// Allocate some coding blobs and insert into the blobs array
pub fn add_coding_blobs(recycler: &BlobRecycler, blobs: &mut Vec<SharedBlob>, consumed: u64) {
let num_data_segments = blobs.len() / NUM_DATA;
trace!(
"num_data: {} blobs.len(): {}",
num_data_segments,
blobs.len()
);
for i in 0..num_data_segments {
let idx = (i * NUM_CODED) + NUM_DATA - (consumed as usize) % NUM_CODED;
for j in idx..idx + MAX_MISSING {
trace!("putting coding at {}", j);
if j <= blobs.len() {
let new_blob = recycler.allocate();
blobs.insert(j, new_blob);
}
}
}
}
// Generate coding blocks in window starting from consumed
pub fn generate_coding(window: &mut Vec<Option<SharedBlob>>, consumed: usize) -> Result<()> {
let mut data_blobs = Vec::new();
let mut coding_blobs = Vec::new();
let mut data_locks = Vec::new();
let mut data_ptrs: Vec<&[u8]> = Vec::new();
let mut coding_locks = Vec::new();
let mut coding_ptrs: Vec<&mut [u8]> = Vec::new();
for i in consumed..consumed + NUM_DATA {
let block_start = consumed - (consumed % NUM_CODED);
trace!(
"generate start: {} end: {}",
block_start,
block_start + NUM_DATA
);
for i in block_start..block_start + NUM_DATA {
let n = i % window.len();
trace!("window[{}] = {:?}", n, window[n]);
if window[n].is_none() {
trace!("data block is null @ {}", n);
return Ok(());
}
data_blobs.push(
window[n]
.clone()
@@ -179,11 +207,14 @@ pub fn generate_coding(
}
// generate coding ptr array
let coding_start = consumed + NUM_DATA;
let coding_end = consumed + NUM_CODED;
let coding_start = block_start + NUM_DATA;
let coding_end = block_start + NUM_CODED;
for i in coding_start..coding_end {
let n = i % window.len();
window[n] = Some(re.allocate());
if window[n].is_none() {
trace!("coding block is null @ {}", n);
return Ok(());
}
coding_blobs.push(
window[n]
.clone()
@@ -197,7 +228,7 @@ pub fn generate_coding(
);
}
for (i, l) in coding_locks.iter_mut().enumerate() {
trace!("i: {} data: {}", i, l.data[0]);
trace!("i: {} coding: {}", i, l.data[0]);
coding_ptrs.push(&mut l.data);
}
@@ -218,9 +249,16 @@ pub fn recover(
//recover with erasure coding
let mut data_missing = 0;
let mut coded_missing = 0;
let coding_start = consumed + NUM_DATA;
let coding_end = consumed + NUM_CODED;
for i in consumed..coding_end {
let block_start = consumed - (consumed % NUM_CODED);
let coding_start = block_start + NUM_DATA;
let coding_end = block_start + NUM_CODED;
trace!(
"block_start: {} coding_start: {} coding_end: {}",
block_start,
coding_start,
coding_end
);
for i in block_start..coding_end {
let n = i % window.len();
if window[n].is_none() {
if i >= coding_start {
@@ -238,7 +276,7 @@ pub fn recover(
let mut data_ptrs: Vec<&mut [u8]> = Vec::new();
let mut coding_ptrs: Vec<&[u8]> = Vec::new();
let mut erasures: Vec<i32> = Vec::new();
for i in consumed..coding_end {
for i in block_start..coding_end {
let j = i % window.len();
let mut b = &mut window[j];
if b.is_some() {
@@ -249,7 +287,7 @@ pub fn recover(
*b = Some(n.clone());
//mark the missing memory
blobs.push(n);
erasures.push((i - consumed) as i32);
erasures.push(i as i32);
}
erasures.push(-1);
trace!("erasures: {:?}", erasures);
@@ -282,7 +320,8 @@ pub fn recover(
#[cfg(test)]
mod test {
use erasure;
use packet::{BlobRecycler, SharedBlob, PACKET_DATA_SIZE};
use logger;
use packet::{BlobRecycler, SharedBlob};
#[test]
pub fn test_coding() {
@@ -350,45 +389,79 @@ mod test {
}
}
#[test]
pub fn test_window_recover() {
let mut window = Vec::new();
let blob_recycler = BlobRecycler::default();
let offset = 4;
for i in 0..(4 * erasure::NUM_CODED + 1) {
fn generate_window(
data_len: usize,
blob_recycler: &BlobRecycler,
offset: usize,
) -> Vec<Option<SharedBlob>> {
let mut window = vec![None; 16];
let mut blobs = Vec::new();
for i in 0..erasure::NUM_DATA + 2 {
let b = blob_recycler.allocate();
let b_ = b.clone();
let data_len = b.read().unwrap().data.len();
let mut w = b.write().unwrap();
w.set_index(i as u64).unwrap();
assert_eq!(i as u64, w.get_index().unwrap());
w.meta.size = PACKET_DATA_SIZE;
w.meta.size = data_len;
for k in 0..data_len {
w.data[k] = (k + i) as u8;
w.data_mut()[k] = (k + i) as u8;
}
window.push(Some(b_));
blobs.push(b_);
}
erasure::add_coding_blobs(blob_recycler, &mut blobs, offset as u64);
for (i, b) in blobs.into_iter().enumerate() {
window[i] = Some(b);
}
window
}
#[test]
pub fn test_window_recover_basic() {
logger::setup();
let data_len = 16;
let blob_recycler = BlobRecycler::default();
// Generate a window
let offset = 4;
let mut window = generate_window(data_len, &blob_recycler, 0);
println!("** after-gen-window:");
print_window(&window);
// Generate the coding blocks
assert!(erasure::generate_coding(&mut window, offset).is_ok());
println!("** after-gen-coding:");
print_window(&window);
// Create a hole in the window
let refwindow = window[offset + 1].clone();
window[offset + 1] = None;
// Recover it from coding
assert!(erasure::recover(&blob_recycler, &mut window, offset).is_ok());
println!("** after-recover:");
print_window(&window);
// Check the result
let window_l = window[offset + 1].clone().unwrap();
let ref_l = refwindow.clone().unwrap();
assert_eq!(
window_l.read().unwrap().data()[..data_len],
ref_l.read().unwrap().data()[..data_len]
);
}
//TODO This needs to be reworked
#[test]
#[ignore]
pub fn test_window_recover() {
logger::setup();
let blob_recycler = BlobRecycler::default();
let offset = 4;
let data_len = 16;
let mut window = generate_window(data_len, &blob_recycler, offset);
println!("** after-gen:");
print_window(&window);
assert!(erasure::generate_coding(&blob_recycler, &mut window, offset).is_ok());
assert!(
erasure::generate_coding(&blob_recycler, &mut window, offset + erasure::NUM_CODED)
.is_ok()
);
assert!(
erasure::generate_coding(
&blob_recycler,
&mut window,
offset + (2 * erasure::NUM_CODED)
).is_ok()
);
assert!(
erasure::generate_coding(
&blob_recycler,
&mut window,
offset + (3 * erasure::NUM_CODED)
).is_ok()
);
assert!(erasure::generate_coding(&mut window, offset).is_ok());
println!("** after-coding:");
print_window(&window);
let refwindow = window[offset + 1].clone();
@@ -403,28 +476,13 @@ mod test {
println!("** after-nulling:");
print_window(&window);
assert!(erasure::recover(&blob_recycler, &mut window, offset).is_ok());
assert!(erasure::recover(&blob_recycler, &mut window, offset + erasure::NUM_CODED).is_ok());
assert!(
erasure::recover(
&blob_recycler,
&mut window,
offset + (2 * erasure::NUM_CODED)
).is_err()
);
assert!(
erasure::recover(
&blob_recycler,
&mut window,
offset + (3 * erasure::NUM_CODED)
).is_ok()
);
println!("** after-restore:");
print_window(&window);
let window_l = window[offset + 1].clone().unwrap();
let ref_l = refwindow.clone().unwrap();
assert_eq!(
window_l.read().unwrap().data.to_vec(),
ref_l.read().unwrap().data.to_vec()
window_l.read().unwrap().data()[..data_len],
ref_l.read().unwrap().data()[..data_len]
);
}
}