gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

rework erasure to have data and coding blobs side-by-side in window

Browse Source
This commit is contained in:
Rob Walker
2018-07-17 13:02:38 -07:00
parent dda563a169
commit 99653a4d04
1 changed files with 239 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files

440
src/erasure.rs
View File

@ -1,12 +1,12 @@
// Support erasure coding // Support erasure coding
use packet::{BlobRecycler, SharedBlob, BLOB_HEADER_SIZE}; use packet::{BlobRecycler, SharedBlob, BLOB_HEADER_SIZE};
use std::result; use std::result;
use streamer::WindowSlot;
//TODO(sakridge) pick these values //TODO(sakridge) pick these values
pub const NUM_CODED: usize = 20; pub const NUM_DATA: usize = 16; // number of data blobs
pub const MAX_MISSING: usize = 4; pub const NUM_CODING: usize = 4; // number of coded blobs, also the maximum number that can go missing
const NUM_DATA: usize = NUM_CODED - MAX_MISSING; pub const ERASURE_SET_SIZE: usize = NUM_DATA + NUM_CODING; // total number of blobs in an erasure set, includes data and coded blobs
#[derive(Debug, PartialEq, Eq)] #[derive(Debug, PartialEq, Eq)]
pub enum ErasureError { pub enum ErasureError {
@ -67,13 +67,13 @@ pub fn generate_coding_blocks(coding: &mut [&mut [u8]], data: &[&[u8]]) -> Resul
if data.len() == 0 { if data.len() == 0 {
return Ok(()); return Ok(());
} }
let k = data.len() as i32;
let m = coding.len() as i32; let m = coding.len() as i32;
let block_len = data[0].len(); let block_len = data[0].len() as i32;
let matrix: Vec<i32> = get_matrix(m, data.len() as i32, ERASURE_W); let matrix: Vec<i32> = get_matrix(m, k, ERASURE_W);
let mut coding_arg = Vec::new(); let mut data_arg = Vec::with_capacity(data.len());
let mut data_arg = Vec::new();
for block in data { for block in data {
if block_len != block.len() { if block_len != block.len() as i32 {
trace!( trace!(
"data block size incorrect {} expected {}", "data block size incorrect {} expected {}",
block.len(), block.len(),
@ -83,8 +83,9 @@ pub fn generate_coding_blocks(coding: &mut [&mut [u8]], data: &[&[u8]]) -> Resul
} }
data_arg.push(block.as_ptr()); data_arg.push(block.as_ptr());
} }
let mut coding_arg = Vec::with_capacity(coding.len());
for mut block in coding { for mut block in coding {
if block_len != block.len() { if block_len != block.len() as i32 {
trace!( trace!(
"coding block size incorrect {} expected {}", "coding block size incorrect {} expected {}",
block.len(), block.len(),
@ -97,13 +98,13 @@ pub fn generate_coding_blocks(coding: &mut [&mut [u8]], data: &[&[u8]]) -> Resul
unsafe { unsafe {
jerasure_matrix_encode( jerasure_matrix_encode(
data.len() as i32, k,
m, m,
ERASURE_W, ERASURE_W,
matrix.as_ptr(), matrix.as_ptr(),
data_arg.as_ptr(), data_arg.as_ptr(),
coding_arg.as_ptr(), coding_arg.as_ptr(),
data[0].len() as i32, block_len,
); );
} }
Ok(()) Ok(())
@ -162,49 +163,57 @@ pub fn decode_blocks(data: &mut [&mut [u8]], coding: &[&[u8]], erasures: &[i32])
} }
// Allocate some coding blobs and insert into the blobs array // Allocate some coding blobs and insert into the blobs array
pub fn add_coding_blobs(recycler: &BlobRecycler, blobs: &mut Vec<SharedBlob>, consumed: u64) { //pub fn add_coding_blobs(recycler: &BlobRecycler, blobs: &mut [WindowSlot], consumed: u64) {
let mut added = 0; // let mut added = 0;
let blobs_len = blobs.len() as u64; // let blobs_len = blobs.len() as u64;
for i in consumed..consumed + blobs_len { // for i in consumed..consumed + blobs_len {
let is = i as usize; // let is = i as usize;
if is != 0 && ((is + MAX_MISSING) % NUM_CODED) == 0 { // if is != 0 && ((is + NUM_CODING) % NUM_DATA) == 0 {
for _ in 0..MAX_MISSING { // for _ in 0..NUM_CODING {
trace!("putting coding at {}", (i - consumed)); // trace!("putting coding at {}", (i - consumed));
let new_blob = recycler.allocate(); // let new_blob = recycler.allocate();
let new_blob_clone = new_blob.clone(); // let new_blob_clone = new_blob.clone();
let mut new_blob_l = new_blob_clone.write().unwrap(); // let mut new_blob_l = new_blob_clone.write().unwrap();
new_blob_l.set_size(0); // new_blob_l.set_size(0);
new_blob_l.set_coding().unwrap(); // new_blob_l.set_coding().unwrap();
drop(new_blob_l); // drop(new_blob_l);
blobs.insert((i - consumed) as usize, new_blob); // blobs.insert((i - consumed) as usize, new_blob);
added += 1; // added += 1;
} // }
} // }
} // }
info!( // info!(
"add_coding consumed: {} blobs.len(): {} added: {}", // "add_coding consumed: {} blobs.len(): {} added: {}",
consumed, // consumed,
blobs.len(), // blobs.len(),
added // added
); // );
} //}
// Generate coding blocks in window starting from consumed // Generate coding blocks in window starting from consumed,
// for each block place the coding blobs at the end of the block
//
// block-size part of a Window, with each element a WindowSlot..
// |<======================= NUM_DATA ==============================>|
// |<==== NUM_CODING ===>|
// +---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+
// | D | | D | | D | | D | | D | | D | | D | | D | | D | | D |
// +---+ +---+ +---+ +---+ +---+ . . . +---+ +---+ +---+ +---+ +---+
// | | | | | | | | | | | | | C | | C | | C | | C |
// +---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+ +---+
pub fn generate_coding( pub fn generate_coding(
window: &mut Vec<Option<SharedBlob>>, window: &mut [WindowSlot],
recycler: &BlobRecycler,
consumed: usize, consumed: usize,
num_blobs: usize, num_blobs: usize,
) -> Result<()> { ) -> Result<()> {
let mut block_start = consumed - (consumed % NUM_CODED); let mut block_start = consumed - (consumed % NUM_DATA);
for i in consumed..consumed + num_blobs { for i in consumed..consumed + num_blobs {
if (i % NUM_CODED) == (NUM_CODED - 1) { if (i % NUM_DATA) == (NUM_DATA - 1) {
let mut data_blobs = Vec::new(); let mut data_blobs = Vec::with_capacity(NUM_DATA);
let mut coding_blobs = Vec::new(); let mut data_locks = Vec::with_capacity(NUM_DATA);
let mut data_locks = Vec::new(); let mut data_ptrs: Vec<&[u8]> = Vec::with_capacity(NUM_DATA);
let mut data_ptrs: Vec<&[u8]> = Vec::new();
let mut coding_locks = Vec::new();
let mut coding_ptrs: Vec<&mut [u8]> = Vec::new();
info!( info!(
"generate_coding start: {} end: {} consumed: {} num_blobs: {}", "generate_coding start: {} end: {} consumed: {} num_blobs: {}",
@ -215,13 +224,14 @@ pub fn generate_coding(
); );
for i in block_start..block_start + NUM_DATA { for i in block_start..block_start + NUM_DATA {
let n = i % window.len(); let n = i % window.len();
trace!("window[{}] = {:?}", n, window[n]); trace!("window[{}] = {:?}", n, window[n].data);
if window[n].is_none() { if window[n].data.is_none() {
trace!("data block is null @ {}", n); trace!("data block is null @ {}", n);
return Ok(()); return Ok(());
} }
data_blobs.push( data_blobs.push(
window[n] window[n]
.data
.clone() .clone()
.expect("'data_blobs' arr in pub fn generate_coding"), .expect("'data_blobs' arr in pub fn generate_coding"),
); );
@ -240,22 +250,33 @@ pub fn generate_coding(
data_ptrs.push(&l.data[..max_data_size]); data_ptrs.push(&l.data[..max_data_size]);
} }
// generate coding ptr array let mut coding_blobs = Vec::with_capacity(NUM_CODING);
let coding_start = block_start + NUM_DATA; let mut coding_locks = Vec::with_cpacity(NUM_CODING);
let coding_end = block_start + NUM_CODED; let mut coding_ptrs: Vec<&mut [u8]> = Vec::with_cpacity(NUM_CODING);
let coding_start = block_start + NUM_DATA - NUM_CODING;
let coding_end = block_start + NUM_DATA;
for i in coding_start..coding_end { for i in coding_start..coding_end {
let n = i % window.len(); let n = i % window.len();
if window[n].is_none() { if window[n].coding.is_none() {
trace!("coding block is null @ {}", n); window[n].coding = Some(recycler.allocate());
return Ok(());
} }
let w_l = window[n].clone().unwrap();
let w_l = window[n].coding.clone().unwrap();
w_l.write().unwrap().set_size(max_data_size); w_l.write().unwrap().set_size(max_data_size);
w_l.write()
.unwrap()
.set_index(window[n].data.get_index().unwrap());
w_l.write()
.unwrap()
.set_id(window[n].data.get_id().unwrap());
if w_l.write().unwrap().set_coding().is_err() { if w_l.write().unwrap().set_coding().is_err() {
return Err(ErasureError::EncodeError); return Err(ErasureError::EncodeError);
} }
coding_blobs.push( coding_blobs.push(
window[n] window[n]
.coding
.clone() .clone()
.expect("'coding_blobs' arr in pub fn generate_coding"), .expect("'coding_blobs' arr in pub fn generate_coding"),
); );
@ -280,7 +301,7 @@ pub fn generate_coding(
coding_start, coding_start,
coding_end coding_end
); );
block_start += NUM_CODED; block_start += NUM_DATA;
} }
} }
Ok(()) Ok(())
@ -291,8 +312,8 @@ pub fn generate_coding(
// to allocate new ones. Returns err if not enough // to allocate new ones. Returns err if not enough
// coding blocks are present to restore // coding blocks are present to restore
pub fn recover( pub fn recover(
re: &BlobRecycler, recycler: &BlobRecycler,
window: &mut Vec<Option<SharedBlob>>, window: &mut [WindowSlot],
consumed: usize, consumed: usize,
received: usize, received: usize,
) -> Result<()> { ) -> Result<()> {
@ -300,8 +321,8 @@ pub fn recover(
if received <= consumed { if received <= consumed {
return Ok(()); return Ok(());
} }
let num_blocks = (received - consumed) / NUM_CODED; let num_blocks = (received - consumed) / NUM_DATA;
let mut block_start = consumed - (consumed % NUM_CODED); let mut block_start = consumed - (consumed % NUM_DATA);
if num_blocks > 0 { if num_blocks > 0 {
debug!( debug!(
@ -315,9 +336,9 @@ pub fn recover(
break; break;
} }
let mut data_missing = 0; let mut data_missing = 0;
let mut coded_missing = 0; let mut coding_missing = 0;
let coding_start = block_start + NUM_DATA; let coding_start = block_start + NUM_DATA - NUM_CODING;
let coding_end = block_start + NUM_CODED; let coding_end = block_start + NUM_DATA;
trace!( trace!(
"recover: block_start: {} coding_start: {} coding_end: {}", "recover: block_start: {} coding_start: {} coding_end: {}",
block_start, block_start,
@ -326,98 +347,113 @@ pub fn recover(
); );
for i in block_start..coding_end { for i in block_start..coding_end {
let n = i % window.len(); let n = i % window.len();
if window[n].is_none() { if window[n].coding.is_none() && i >= coding_start {
if i >= coding_start { coding_missing += 1;
coded_missing += 1; }
} else { if window[n].data.is_none() {
data_missing += 1; data_missing += 1;
}
} }
} }
if (data_missing + coded_missing) != NUM_CODED && (data_missing + coded_missing) != 0 {
// if we're not missing data, or if we have too much missin but have enough coding
if data_missing == 0 || (data_missing + coding_missing) > NUM_CODING {
debug!( debug!(
"1: start: {} recovering: data: {} coding: {}", "1: start: {} skipping recovery data: {} coding: {}",
block_start, data_missing, coded_missing block_start, data_missing, coding_missing
);
block_start += NUM_DATA;
continue;
}
debug!(
"2: recovering: data: {} coding: {}",
data_missing, coding_missing
);
let mut blobs: Vec<SharedBlob> = Vec::with_capacity(NUM_DATA + NUM_CODING);
let mut locks = Vec::with_capacity(NUM_DATA + NUM_CODING);
let mut erasures: Vec<i32> = Vec::with_capacity(NUM_CODING);
let mut meta = None;
let mut size = None;
// add the data blobs we have into recovery blob vector
for i in block_start..coding_end {
let j = i % window.len();
let mut b = &mut window[j];
if b.data.is_some() {
if meta.is_none() {
let bl = b.data.clone().unwrap();
meta = Some(bl.read().unwrap().meta.clone());
}
blobs.push(b.data.clone().expect("'blobs' arr in pb fn recover"));
} else {
let n = recycler.allocate();
*b.data = Some(n.clone());
// mark the missing memory
blobs.push(n);
erasures.push((i - block_start) as i32);
}
}
for i in coding_start..coding_end {
let j = i % window.len();
let mut b = &mut window[j];
if b.coding.is_some() {
if size.is_none() {
let bl = b.coding.clone().unwrap();
size = Some(bl.read().unwrap().meta.size - BLOB_HEADER_SIZE);
}
blobs.push(b.coding.clone().expect("'blobs' arr in pb fn recover"));
} else {
let n = recycler.allocate();
*b = Some(n.clone());
//mark the missing memory
blobs.push(n);
erasures.push((i - block_start + NUM_DATA) as i32);
}
}
erasures.push(-1);
trace!(
"erasures: {:?} data_size: {} header_size: {}",
erasures,
size.unwrap(),
BLOB_HEADER_SIZE
);
//lock everything
for b in &blobs {
locks.push(b.write().expect("'locks' arr in pb fn recover"));
}
{
let mut coding_ptrs: Vec<&[u8]> = Vec::with_capacity(NUM_CODING);
let mut data_ptrs: Vec<&mut [u8]> = Vec::with_capacity(NUM_DATA);
for (i, l) in locks.iter_mut().enumerate() {
if i >= NUM_DATA {
trace!("pushing coding: {}", i);
coding_ptrs.push(&l.data()[..size.unwrap()]);
} else {
trace!("pushing data: {}", i);
data_ptrs.push(&mut l.data[..size.unwrap()]);
}
}
trace!(
"coding_ptrs.len: {} data_ptrs.len {}",
coding_ptrs.len(),
data_ptrs.len()
);
decode_blocks(data_ptrs.as_mut_slice(), &coding_ptrs, &erasures)?;
}
for i in &erasures[..erasures.len() - 1] {
let idx = *i as usize;
let data_size = locks[idx].get_data_size().unwrap() - BLOB_HEADER_SIZE as u64;
locks[idx].meta = meta.clone().unwrap();
locks[idx].set_size(data_size as usize);
trace!(
"erasures[{}] size: {} data[0]: {}",
*i,
data_size,
locks[idx].data()[0]
); );
} }
if data_missing > 0 { block_start += NUM_DATA;
if (data_missing + coded_missing) <= MAX_MISSING {
debug!(
"2: recovering: data: {} coding: {}",
data_missing, coded_missing
);
let mut blobs: Vec<SharedBlob> = Vec::new();
let mut locks = Vec::new();
let mut erasures: Vec<i32> = Vec::new();
let mut meta = None;
let mut size = None;
for i in block_start..coding_end {
let j = i % window.len();
let mut b = &mut window[j];
if b.is_some() {
if i >= NUM_DATA && size.is_none() {
let bl = b.clone().unwrap();
size = Some(bl.read().unwrap().meta.size - BLOB_HEADER_SIZE);
}
if meta.is_none() {
let bl = b.clone().unwrap();
meta = Some(bl.read().unwrap().meta.clone());
}
blobs.push(b.clone().expect("'blobs' arr in pb fn recover"));
continue;
}
let n = re.allocate();
*b = Some(n.clone());
//mark the missing memory
blobs.push(n);
erasures.push((i - block_start) as i32);
}
erasures.push(-1);
trace!(
"erasures: {:?} data_size: {} header_size: {}",
erasures,
size.unwrap(),
BLOB_HEADER_SIZE
);
//lock everything
for b in &blobs {
locks.push(b.write().expect("'locks' arr in pb fn recover"));
}
{
let mut coding_ptrs: Vec<&[u8]> = Vec::new();
let mut data_ptrs: Vec<&mut [u8]> = Vec::new();
for (i, l) in locks.iter_mut().enumerate() {
if i >= NUM_DATA {
trace!("pushing coding: {}", i);
coding_ptrs.push(&l.data()[..size.unwrap()]);
} else {
trace!("pushing data: {}", i);
data_ptrs.push(&mut l.data[..size.unwrap()]);
}
}
trace!(
"coding_ptrs.len: {} data_ptrs.len {}",
coding_ptrs.len(),
data_ptrs.len()
);
decode_blocks(data_ptrs.as_mut_slice(), &coding_ptrs, &erasures)?;
}
for i in &erasures[..erasures.len() - 1] {
let idx = *i as usize;
let data_size = locks[idx].get_data_size().unwrap() - BLOB_HEADER_SIZE as u64;
locks[idx].meta = meta.clone().unwrap();
locks[idx].set_size(data_size as usize);
trace!(
"erasures[{}] size: {} data[0]: {}",
*i,
data_size,
locks[idx].data()[0]
);
}
}
}
block_start += NUM_CODED;
} }
Ok(()) Ok(())
} }
@ -426,10 +462,11 @@ mod test {
use crdt; use crdt;
use erasure; use erasure;
use logger; use logger;
use packet::{BlobRecycler, SharedBlob, BLOB_HEADER_SIZE}; use packet::{BlobRecycler, BLOB_HEADER_SIZE};
use signature::KeyPair; use signature::KeyPair;
use signature::KeyPairUtil; use signature::KeyPairUtil;
use std::sync::{Arc, RwLock}; // use std::sync::{Arc, RwLock};
use streamer::WindowSlot;
#[test] #[test]
pub fn test_coding() { pub fn test_coding() {
@ -481,11 +518,11 @@ mod test {
assert_eq!(v_orig, vs[0]); assert_eq!(v_orig, vs[0]);
} }
fn print_window(window: &Vec<Option<SharedBlob>>) { fn print_window(window: &[WindowSlot]) {
for (i, w) in window.iter().enumerate() { for (i, w) in window.iter().enumerate() {
print!("window({}): ", i); print!("window({}): ", i);
if w.is_some() { if w.data.is_some() {
let window_l1 = w.clone().unwrap(); let window_l1 = w.data.clone().unwrap();
let window_l2 = window_l1.read().unwrap(); let window_l2 = window_l1.read().unwrap();
print!( print!(
"index: {:?} meta.size: {} data: ", "index: {:?} meta.size: {} data: ",
@ -507,8 +544,11 @@ mod test {
blob_recycler: &BlobRecycler, blob_recycler: &BlobRecycler,
offset: usize, offset: usize,
num_blobs: usize, num_blobs: usize,
) -> (Vec<Option<SharedBlob>>, usize) { ) -> [WindowSlot; 32] {
let mut window = vec![None; 32]; let mut window = [WindowSlot {
data: None,
coding: None,
}; 32];
let mut blobs = Vec::new(); let mut blobs = Vec::new();
for i in 0..num_blobs { for i in 0..num_blobs {
let b = blob_recycler.allocate(); let b = blob_recycler.allocate();
@ -520,8 +560,6 @@ mod test {
} }
blobs.push(b_); blobs.push(b_);
} }
erasure::add_coding_blobs(blob_recycler, &mut blobs, offset as u64);
let blobs_len = blobs.len();
let d = crdt::NodeInfo::new( let d = crdt::NodeInfo::new(
KeyPair::new().pubkey(), KeyPair::new().pubkey(),
@ -534,9 +572,9 @@ mod test {
assert!(crdt::Crdt::index_blobs(&d, &blobs, &mut (offset as u64)).is_ok()); assert!(crdt::Crdt::index_blobs(&d, &blobs, &mut (offset as u64)).is_ok());
for b in blobs { for b in blobs {
let idx = b.read().unwrap().get_index().unwrap() as usize; let idx = b.read().unwrap().get_index().unwrap() as usize;
window[idx] = Some(b); window[idx].data = Some(b);
} }
(window, blobs_len) window
} }
#[test] #[test]
@ -548,12 +586,12 @@ mod test {
// Generate a window // Generate a window
let offset = 1; let offset = 1;
let num_blobs = erasure::NUM_DATA + 2; let num_blobs = erasure::NUM_DATA + 2;
let (mut window, blobs_len) = generate_window(data_len, &blob_recycler, 0, num_blobs); let mut window = generate_window(data_len, &blob_recycler, 0, num_blobs);
println!("** after-gen-window:"); println!("** after-gen-window:");
print_window(&window); print_window(&window);
// Generate the coding blocks // Generate the coding blocks
assert!(erasure::generate_coding(&mut window, offset, blobs_len).is_ok()); assert!(erasure::generate_coding(&mut window, blob_recycler, offset, num_blobs).is_ok());
println!("** after-gen-coding:"); println!("** after-gen-coding:");
print_window(&window); print_window(&window);
@ -563,7 +601,7 @@ mod test {
window[erase_offset] = None; window[erase_offset] = None;
// Recover it from coding // Recover it from coding
assert!(erasure::recover(&blob_recycler, &mut window, offset, offset + blobs_len).is_ok()); assert!(erasure::recover(&blob_recycler, &mut window, offset, offset + num_blobs).is_ok());
println!("** after-recover:"); println!("** after-recover:");
print_window(&window); print_window(&window);
@ -583,40 +621,40 @@ mod test {
assert_eq!(window_l2.get_index().unwrap(), erase_offset as u64); assert_eq!(window_l2.get_index().unwrap(), erase_offset as u64);
} }
//TODO This needs to be reworked // //TODO This needs to be reworked
#[test] // #[test]
#[ignore] // #[ignore]
pub fn test_window_recover() { // pub fn test_window_recover() {
logger::setup(); // logger::setup();
let blob_recycler = BlobRecycler::default(); // let blob_recycler = BlobRecycler::default();
let offset = 4; // let offset = 4;
let data_len = 16; // let data_len = 16;
let num_blobs = erasure::NUM_DATA + 2; // let num_blobs = erasure::NUM_DATA + 2;
let (mut window, blobs_len) = generate_window(data_len, &blob_recycler, offset, num_blobs); // let (mut window, blobs_len) = generate_window(data_len, &blob_recycler, offset, num_blobs);
println!("** after-gen:"); // println!("** after-gen:");
print_window(&window); // print_window(&window);
assert!(erasure::generate_coding(&mut window, offset, blobs_len).is_ok()); // assert!(erasure::generate_coding(&mut window, offset, blobs_len).is_ok());
println!("** after-coding:"); // println!("** after-coding:");
print_window(&window); // print_window(&window);
let refwindow = window[offset + 1].clone(); // let refwindow = window[offset + 1].clone();
window[offset + 1] = None; // window[offset + 1] = None;
window[offset + 2] = None; // window[offset + 2] = None;
window[offset + erasure::NUM_CODED + 3] = None; // window[offset + erasure::SET_SIZE + 3] = None;
window[offset + (2 * erasure::NUM_CODED) + 0] = None; // window[offset + (2 * erasure::SET_SIZE) + 0] = None;
window[offset + (2 * erasure::NUM_CODED) + 1] = None; // window[offset + (2 * erasure::SET_SIZE) + 1] = None;
window[offset + (2 * erasure::NUM_CODED) + 2] = None; // window[offset + (2 * erasure::SET_SIZE) + 2] = None;
let window_l0 = &(window[offset + (3 * erasure::NUM_CODED)]).clone().unwrap(); // let window_l0 = &(window[offset + (3 * erasure::SET_SIZE)]).clone().unwrap();
window_l0.write().unwrap().data[0] = 55; // window_l0.write().unwrap().data[0] = 55;
println!("** after-nulling:"); // println!("** after-nulling:");
print_window(&window); // print_window(&window);
assert!(erasure::recover(&blob_recycler, &mut window, offset, offset + blobs_len).is_ok()); // assert!(erasure::recover(&blob_recycler, &mut window, offset, offset + blobs_len).is_ok());
println!("** after-restore:"); // println!("** after-restore:");
print_window(&window); // print_window(&window);
let window_l = window[offset + 1].clone().unwrap(); // let window_l = window[offset + 1].clone().unwrap();
let ref_l = refwindow.clone().unwrap(); // let ref_l = refwindow.clone().unwrap();
assert_eq!( // assert_eq!(
window_l.read().unwrap().data()[..data_len], // window_l.read().unwrap().data()[..data_len],
ref_l.read().unwrap().data()[..data_len] // ref_l.read().unwrap().data()[..data_len]
); // );
} // }
} }