gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rework to fix coding blob insertion

Browse Source
This commit is contained in:
Stephen Akridge
2018-06-05 09:50:50 -07:00
committed by Greg Fitzgerald
parent ec3569aa39
commit cb16fe84cd
1 changed files with 99 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
src/erasure.rs
View File

@ -4,8 +4,8 @@ use packet::{BlobRecycler, SharedBlob, BLOB_HEADER_SIZE};
use std::result; use std::result;
//TODO(sakridge) pick these values //TODO(sakridge) pick these values
const NUM_CODED: usize = 3; const NUM_CODED: usize = 20;
const MAX_MISSING: usize = 1; const MAX_MISSING: usize = 4;
const NUM_DATA: usize = NUM_CODED - MAX_MISSING; const NUM_DATA: usize = NUM_CODED - MAX_MISSING;
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
@ -155,26 +155,29 @@ pub fn decode_blocks(data: &mut [&mut [u8]], coding: &[&[u8]], erasures: &[i32])
// Allocate some coding blobs and insert into the blobs array // Allocate some coding blobs and insert into the blobs array
pub fn add_coding_blobs(recycler: &BlobRecycler, blobs: &mut Vec<SharedBlob>, consumed: u64) { pub fn add_coding_blobs(recycler: &BlobRecycler, blobs: &mut Vec<SharedBlob>, consumed: u64) {
let num_data_segments = blobs.len() / NUM_DATA; let mut added = 0;
info!( let blobs_len = blobs.len() as u64;
"add_coding num_data: {} blobs.len(): {}", for i in consumed..consumed + blobs_len {
num_data_segments, let is = i as usize;
blobs.len() if is != 0 && ((is + MAX_MISSING) % NUM_CODED) == 0 {
); for _ in 0..MAX_MISSING {
for i in 0..num_data_segments { trace!("putting coding at {}", (i - consumed));
let idx = (i * NUM_CODED) + NUM_DATA - (consumed as usize) % NUM_CODED;
for j in idx..idx + MAX_MISSING {
trace!("putting coding at {}", j);
if j <= blobs.len() {
let new_blob = recycler.allocate(); let new_blob = recycler.allocate();
let new_blob_clone = new_blob.clone(); let new_blob_clone = new_blob.clone();
let mut new_blob_l = new_blob_clone.write().unwrap(); let mut new_blob_l = new_blob_clone.write().unwrap();
new_blob_l.meta.size = new_blob_l.data().len(); new_blob_l.meta.size = new_blob_l.data().len();
drop(new_blob_l); drop(new_blob_l);
blobs.insert(j, new_blob); blobs.insert((i - consumed) as usize, new_blob);
added += 1;
} }
} }
} }
info!(
"add_coding consumed: {} blobs.len(): {} added: {}",
consumed,
blobs.len(),
added
);
} }
// Generate coding blocks in window starting from consumed // Generate coding blocks in window starting from consumed
@ -182,9 +185,8 @@ pub fn generate_coding(window: &mut Vec<Option<SharedBlob>>, consumed: usize, nu
let mut block_start = consumed - (consumed % NUM_CODED); let mut block_start = consumed - (consumed % NUM_CODED);
let num_blocks = num_blobs / NUM_CODED; for i in consumed..consumed + num_blobs {
if i != 0 && (i % (NUM_CODED - 1)) == 0 {
for _ in 0..num_blocks {
let mut data_blobs = Vec::new(); let mut data_blobs = Vec::new();
let mut coding_blobs = Vec::new(); let mut coding_blobs = Vec::new();
@ -194,9 +196,11 @@ pub fn generate_coding(window: &mut Vec<Option<SharedBlob>>, consumed: usize, nu
let mut coding_ptrs: Vec<&mut [u8]> = Vec::new(); let mut coding_ptrs: Vec<&mut [u8]> = Vec::new();
info!( info!(
"generate_coding start: {} end: {}", "generate_coding start: {} end: {} consumed: {} num_blobs: {}",
block_start, block_start,
block_start + NUM_DATA block_start + NUM_DATA,
consumed,
num_blobs
); );
for i in block_start..block_start + NUM_DATA { for i in block_start..block_start + NUM_DATA {
let n = i % window.len(); let n = i % window.len();
@ -257,9 +261,11 @@ pub fn generate_coding(window: &mut Vec<Option<SharedBlob>>, consumed: usize, nu
} }
generate_coding_blocks(coding_ptrs.as_mut_slice(), &data_ptrs)?; generate_coding_blocks(coding_ptrs.as_mut_slice(), &data_ptrs)?;
debug!("consumed: {}", consumed); debug!("consumed: {} data: {}:{} coding: {}:{}", consumed,
block_start, block_start + NUM_DATA, coding_start, coding_end);
block_start += NUM_CODED; block_start += NUM_CODED;
} }
}
Ok(()) Ok(())
} }
@ -460,8 +466,8 @@ mod test {
blob_recycler: &BlobRecycler, blob_recycler: &BlobRecycler,
offset: usize, offset: usize,
num_blobs: usize, num_blobs: usize,
) -> Vec<Option<SharedBlob>> { ) -> (Vec<Option<SharedBlob>>, usize) {
let mut window = vec![None; 16]; let mut window = vec![None; 32];
let mut blobs = Vec::new(); let mut blobs = Vec::new();
for i in 0..num_blobs { for i in 0..num_blobs {
let b = blob_recycler.allocate(); let b = blob_recycler.allocate();
@ -474,6 +480,7 @@ mod test {
blobs.push(b_); blobs.push(b_);
} }
erasure::add_coding_blobs(blob_recycler, &mut blobs, offset as u64); erasure::add_coding_blobs(blob_recycler, &mut blobs, offset as u64);
let blobs_len = blobs.len();
let d = crdt::ReplicatedData::new( let d = crdt::ReplicatedData::new(
KeyPair::new().pubkey(), KeyPair::new().pubkey(),
@ -490,7 +497,7 @@ mod test {
let idx = b.read().unwrap().get_index().unwrap() as usize; let idx = b.read().unwrap().get_index().unwrap() as usize;
window[idx] = Some(b); window[idx] = Some(b);
} }
window (window, blobs_len)
} }
#[test] #[test]
@ -502,12 +509,12 @@ mod test {
// Generate a window // Generate a window
let offset = 1; let offset = 1;
let num_blobs = erasure::NUM_DATA + 2; let num_blobs = erasure::NUM_DATA + 2;
let mut window = generate_window(data_len, &blob_recycler, 0, num_blobs); let (mut window, blobs_len) = generate_window(data_len, &blob_recycler, 0, num_blobs);
println!("** after-gen-window:"); println!("** after-gen-window:");
print_window(&window); print_window(&window);
// Generate the coding blocks // Generate the coding blocks
assert!(erasure::generate_coding(&mut window, offset, num_blobs).is_ok()); assert!(erasure::generate_coding(&mut window, offset, blobs_len).is_ok());
println!("** after-gen-coding:"); println!("** after-gen-coding:");
print_window(&window); print_window(&window);
@ -517,7 +524,7 @@ mod test {
window[erase_offset] = None; window[erase_offset] = None;
// Recover it from coding // Recover it from coding
assert!(erasure::recover(&blob_recycler, &mut window, offset, offset + num_blobs).is_ok()); assert!(erasure::recover(&blob_recycler, &mut window, offset, offset + blobs_len).is_ok());
println!("** after-recover:"); println!("** after-recover:");
print_window(&window); print_window(&window);
@ -546,10 +553,10 @@ mod test {
let offset = 4; let offset = 4;
let data_len = 16; let data_len = 16;
let num_blobs = erasure::NUM_DATA + 2; let num_blobs = erasure::NUM_DATA + 2;
let mut window = generate_window(data_len, &blob_recycler, offset, num_blobs); let (mut window, blobs_len) = generate_window(data_len, &blob_recycler, offset, num_blobs);
println!("** after-gen:"); println!("** after-gen:");
print_window(&window); print_window(&window);
assert!(erasure::generate_coding(&mut window, offset, num_blobs).is_ok()); assert!(erasure::generate_coding(&mut window, offset, blobs_len).is_ok());
println!("** after-coding:"); println!("** after-coding:");
print_window(&window); print_window(&window);
let refwindow = window[offset + 1].clone(); let refwindow = window[offset + 1].clone();
@ -563,7 +570,7 @@ mod test {
window_l0.write().unwrap().data[0] = 55; window_l0.write().unwrap().data[0] = 55;
println!("** after-nulling:"); println!("** after-nulling:");
print_window(&window); print_window(&window);
assert!(erasure::recover(&blob_recycler, &mut window, offset, offset + num_blobs).is_ok()); assert!(erasure::recover(&blob_recycler, &mut window, offset, offset + blobs_len).is_ok());
println!("** after-restore:"); println!("** after-restore:");
print_window(&window); print_window(&window);
let window_l = window[offset + 1].clone().unwrap(); let window_l = window[offset + 1].clone().unwrap();