diff --git a/core/src/blocktree.rs b/core/src/blocktree.rs index 5a98116515..fe0bc4e28c 100644 --- a/core/src/blocktree.rs +++ b/core/src/blocktree.rs @@ -1259,6 +1259,7 @@ fn try_erasure_recover( ) -> Result, Vec)>> { use crate::erasure::ERASURE_SET_SIZE; + let set_index = erasure_meta.set_index; let blobs = match erasure_meta.status() { ErasureMetaStatus::CanRecover => { let erasure_result = recover( @@ -1275,20 +1276,21 @@ fn try_erasure_recover( let recovered = data.len() + coding.len(); assert_eq!( ERASURE_SET_SIZE, - recovered - + (erasure_meta.coding.count_ones() + erasure_meta.data.count_ones()) - as usize, + recovered + (erasure_meta.num_coding() + erasure_meta.num_data()) as usize, "Recovery should always complete a set" ); - info!("[try_erasure] recovered {} blobs", recovered); + debug!( + "[try_erasure] slot: {}, set_index: {}, recovered {} blobs", + slot, set_index, recovered + ); inc_new_counter_info!("blocktree-erasure-blobs_recovered", recovered); Some((data, coding)) } Err(Error::ErasureError(e)) => { inc_new_counter_info!("blocktree-erasure-recovery_failed", 1); error!( - "[try_erasure] recovery failed: slot: {}, set_index: {}, cause: {}", + "[try_erasure] slot: {}, set_index: {}, recovery failed: cause: {}", slot, erasure_meta.set_index, e ); None @@ -1298,10 +1300,18 @@ fn try_erasure_recover( } } ErasureMetaStatus::StillNeed(needed) => { + debug!( + "[try_erasure] slot: {}, set_index: {}, still need {} blobs", + slot, set_index, needed + ); inc_new_counter_info!("blocktree-erasure-blobs_needed", needed, 0, 1000); None } ErasureMetaStatus::DataFull => { + debug!( + "[try_erasure] slot: {}, set_index: {}, set full", + slot, set_index, + ); inc_new_counter_info!("blocktree-erasure-complete", 1, 0, 1000); None } @@ -2922,13 +2932,14 @@ pub mod tests { #[test] fn test_erasure_meta_accuracy() { + use crate::erasure::ERASURE_SET_SIZE; use ErasureMetaStatus::{DataFull, StillNeed}; let path = get_tmp_ledger_path!(); let blocktree = Blocktree::open(&path).unwrap(); // two erasure sets - let num_blobs = 32; + let num_blobs = NUM_DATA as u64 * 2; let slot = 0; let (blobs, _) = make_slot_entries(slot, 0, num_blobs); @@ -2938,7 +2949,7 @@ pub mod tests { .map(|blob| Arc::new(RwLock::new(blob))) .collect(); - blocktree.write_blobs(&blobs[8..16]).unwrap(); + blocktree.write_blobs(&blobs[..2]).unwrap(); let erasure_meta_opt = blocktree .erasure_meta(slot, 0) @@ -2947,17 +2958,19 @@ pub mod tests { assert!(erasure_meta_opt.is_some()); let erasure_meta = erasure_meta_opt.unwrap(); - assert_eq!(erasure_meta.status(), StillNeed(8)); + let should_need = ERASURE_SET_SIZE - NUM_CODING - 2; + match erasure_meta.status() { + StillNeed(n) => assert_eq!(n, should_need), + _ => panic!("Should still need more blobs"), + }; - blocktree.write_blobs(&blobs[..8]).unwrap(); + blocktree.write_blobs(&blobs[2..NUM_DATA]).unwrap(); let erasure_meta = blocktree .erasure_meta(slot, 0) .expect("DB get must succeed") .unwrap(); - assert_eq!(erasure_meta.data, 0xFFFF); - assert_eq!(erasure_meta.coding, 0x0); assert_eq!(erasure_meta.status(), DataFull); // insert all coding blobs in first set @@ -2977,23 +2990,29 @@ pub mod tests { .expect("DB get must succeed") .unwrap(); - assert_eq!(erasure_meta.data, 0xFFFF); - assert_eq!(erasure_meta.coding, 0x0F); assert_eq!(erasure_meta.status(), DataFull); - // insert 8 of 16 data blobs in 2nd set - blocktree.write_blobs(&blobs[16..24]).unwrap(); + // insert blobs in the 2nd set until recovery should be possible given all coding blobs + let set2 = &blobs[NUM_DATA..]; + let mut end = 1; + let blobs_needed = ERASURE_SET_SIZE - NUM_CODING; + while end < blobs_needed { + blocktree.write_blobs(&set2[end - 1..end]).unwrap(); - let erasure_meta = blocktree - .erasure_meta(slot, 1) - .expect("DB get must succeed") - .unwrap(); + let erasure_meta = blocktree + .erasure_meta(slot, 1) + .expect("DB get must succeed") + .unwrap(); - assert_eq!(erasure_meta.data, 0x00FF); - assert_eq!(erasure_meta.coding, 0x0); - assert_eq!(erasure_meta.status(), StillNeed(8)); + match erasure_meta.status() { + StillNeed(n) => assert_eq!(n, blobs_needed - end), + _ => panic!("Should still need more blobs"), + }; - // insert all coding blobs in 2nd set + end += 1; + } + + // insert all coding blobs in 2nd set. Should trigger recovery let mut coding_generator = CodingGenerator::new(Arc::clone(&blocktree.session)); let coding_blobs = coding_generator.next(&shared_blobs[NUM_DATA..]); @@ -3005,30 +3024,6 @@ pub mod tests { .unwrap(); } - let erasure_meta = blocktree - .erasure_meta(slot, 1) - .expect("DB get must succeed") - .unwrap(); - - assert_eq!(erasure_meta.data, 0x00FF); - assert_eq!(erasure_meta.coding, 0x0F); - assert_eq!(erasure_meta.status(), StillNeed(4)); - - // insert 3 more data blobs in 2nd erasure set. - blocktree.write_blobs(&blobs[24..27]).unwrap(); - - let erasure_meta = blocktree - .erasure_meta(slot, 1) - .expect("DB get must succeed") - .unwrap(); - - assert_eq!(erasure_meta.data, 0x07FF); - assert_eq!(erasure_meta.coding, 0x0F); - assert_eq!(erasure_meta.status(), StillNeed(1)); - - // insert 1 more data blob, should trigger erasure - blocktree.write_blobs(&blobs[28..29]).unwrap(); - let erasure_meta = blocktree .erasure_meta(slot, 1) .expect("DB get must succeed") @@ -3050,8 +3045,6 @@ pub mod tests { .unwrap(); assert_eq!(erasure_meta.status(), ErasureMetaStatus::DataFull); - assert_eq!(erasure_meta.data, 0xFFFF); - assert_eq!(erasure_meta.coding, 0x0); } #[test] @@ -3107,8 +3100,7 @@ pub mod tests { .expect("Erasure Meta should be present") .unwrap(); - assert_eq!(erasure_meta.data, 0xFFFF); - assert_eq!(erasure_meta.coding, 0x0F); + assert_eq!(erasure_meta.status(), ErasureMetaStatus::DataFull); let retrieved_data = blocktree .data_cf @@ -3146,12 +3138,8 @@ pub mod tests { let shared_coding_blobs = coding_generator.next(&data_blobs); assert_eq!(shared_coding_blobs.len(), NUM_CODING); - // Insert data blobs and coding. Not enough to do recovery - blocktree - .write_shared_blobs(&data_blobs[..NUM_DATA - 5]) - .unwrap(); - - for shared_blob in shared_coding_blobs { + // Insert coding blobs except 1 and no data. Not enough to do recovery + for shared_blob in shared_coding_blobs.iter().skip(1) { let blob = shared_blob.read().unwrap(); let size = blob.size() + BLOB_HEADER_SIZE; @@ -3364,9 +3352,9 @@ pub mod tests { // all possibility for recovery should be exhausted assert_eq!(erasure_meta.status(), ErasureMetaStatus::DataFull); // Should have all data - assert_eq!(erasure_meta.data, 0xFFFF); + assert_eq!(erasure_meta.num_data(), NUM_DATA); // Should have all coding - assert_eq!(erasure_meta.coding, 0x0F); + assert_eq!(erasure_meta.num_coding(), NUM_CODING); } } diff --git a/core/src/blocktree/meta.rs b/core/src/blocktree/meta.rs index 2c572ad291..21e3ccf8a8 100644 --- a/core/src/blocktree/meta.rs +++ b/core/src/blocktree/meta.rs @@ -1,4 +1,5 @@ use crate::erasure::{NUM_CODING, NUM_DATA}; +use std::borrow::Borrow; #[derive(Clone, Debug, Default, Deserialize, Serialize, Eq, PartialEq)] // The Meta column family @@ -81,9 +82,9 @@ pub struct ErasureMeta { /// Size of shards in this erasure set pub size: usize, /// Bitfield representing presence/absence of data blobs - pub data: u64, + data: u64, /// Bitfield representing presence/absence of coding blobs - pub coding: u64, + coding: u64, } #[derive(Debug, PartialEq)] @@ -104,10 +105,8 @@ impl ErasureMeta { } pub fn status(&self) -> ErasureMetaStatus { - let (data_missing, coding_missing) = ( - NUM_DATA - self.data.count_ones() as usize, - NUM_CODING - self.coding.count_ones() as usize, - ); + let (data_missing, coding_missing) = + (NUM_DATA - self.num_data(), NUM_CODING - self.num_coding()); if data_missing > 0 && data_missing + coding_missing <= NUM_CODING { assert!(self.size != 0); ErasureMetaStatus::CanRecover @@ -118,6 +117,14 @@ impl ErasureMeta { } } + pub fn num_coding(&self) -> usize { + self.coding.count_ones() as usize + } + + pub fn num_data(&self) -> usize { + self.data.count_ones() as usize + } + pub fn is_coding_present(&self, index: u64) -> bool { if let Some(position) = self.data_index_in_set(index) { self.coding & (1 << position) != 0 @@ -162,6 +169,26 @@ impl ErasureMeta { } } + pub fn set_data_multi(&mut self, indexes: I, present: bool) + where + I: IntoIterator, + Idx: Borrow, + { + for index in indexes.into_iter() { + self.set_data_present(*index.borrow(), present); + } + } + + pub fn set_coding_multi(&mut self, indexes: I, present: bool) + where + I: IntoIterator, + Idx: Borrow, + { + for index in indexes.into_iter() { + self.set_coding_present(*index.borrow(), present); + } + } + pub fn set_index_for(index: u64) -> u64 { index / NUM_DATA as u64 } @@ -201,7 +228,7 @@ fn test_meta_indexes() { for _ in 0..100 { let set_index = rng.gen_range(0, 1_000); - let blob_index = (set_index * NUM_DATA) + rng.gen_range(0, 16); + let blob_index = (set_index * NUM_DATA) + rng.gen_range(0, NUM_DATA); assert_eq!(set_index, ErasureMeta::set_index_for(blob_index)); let e_meta = ErasureMeta::new(set_index); @@ -236,8 +263,8 @@ fn test_meta_indexes() { fn test_meta_coding_present() { let mut e_meta = ErasureMeta::default(); + e_meta.set_coding_multi(0..NUM_CODING as u64, true); for i in 0..NUM_CODING as u64 { - e_meta.set_coding_present(i, true); assert_eq!(e_meta.is_coding_present(i), true); } for i in NUM_CODING as u64..NUM_DATA as u64 { @@ -245,60 +272,62 @@ fn test_meta_coding_present() { } e_meta.set_index = ErasureMeta::set_index_for((NUM_DATA * 17) as u64); + let start_idx = e_meta.start_index(); + e_meta.set_coding_multi(start_idx..start_idx + NUM_CODING as u64, true); - for i in (NUM_DATA * 17) as u64..((NUM_DATA * 17) + NUM_CODING) as u64 { + for i in start_idx..start_idx + NUM_CODING as u64 { e_meta.set_coding_present(i, true); assert_eq!(e_meta.is_coding_present(i), true); } - for i in (NUM_DATA * 17 + NUM_CODING) as u64..((NUM_DATA * 17) + NUM_DATA) as u64 { + for i in start_idx + NUM_CODING as u64..start_idx + NUM_DATA as u64 { assert_eq!(e_meta.is_coding_present(i), false); } } #[test] fn test_erasure_meta_status() { + use rand::{seq::SliceRandom, thread_rng}; + // Local constansts just used to avoid repetitive casts + const N_DATA: u64 = crate::erasure::NUM_DATA as u64; + const N_CODING: u64 = crate::erasure::NUM_CODING as u64; + let mut e_meta = ErasureMeta::default(); + let mut rng = thread_rng(); + let data_indexes: Vec = (0..N_DATA).collect(); + let coding_indexes: Vec = (0..N_CODING).collect(); assert_eq!(e_meta.status(), ErasureMetaStatus::StillNeed(NUM_DATA)); - e_meta.data = 0b1111_1111_1111_1111; - e_meta.coding = 0x00; + e_meta.set_data_multi(0..N_DATA, true); assert_eq!(e_meta.status(), ErasureMetaStatus::DataFull); - e_meta.coding = 0x0e; e_meta.size = 1; + e_meta.set_coding_multi(0..N_CODING, true); + assert_eq!(e_meta.status(), ErasureMetaStatus::DataFull); - e_meta.data = 0b0111_1111_1111_1111; - assert_eq!(e_meta.status(), ErasureMetaStatus::CanRecover); + for &idx in data_indexes.choose_multiple(&mut rng, NUM_CODING) { + e_meta.set_data_present(idx, false); - e_meta.data = 0b0111_1111_1111_1110; - assert_eq!(e_meta.status(), ErasureMetaStatus::CanRecover); + assert_eq!(e_meta.status(), ErasureMetaStatus::CanRecover); + } - e_meta.data = 0b0111_1111_1011_1110; - assert_eq!(e_meta.status(), ErasureMetaStatus::CanRecover); + e_meta.set_data_multi(0..N_DATA, true); - e_meta.data = 0b0111_1011_1011_1110; - assert_eq!(e_meta.status(), ErasureMetaStatus::StillNeed(1)); + for &idx in coding_indexes.choose_multiple(&mut rng, NUM_CODING) { + e_meta.set_coding_present(idx, false); - e_meta.data = 0b0111_1011_1011_1110; - assert_eq!(e_meta.status(), ErasureMetaStatus::StillNeed(1)); - - e_meta.coding = 0b0000_1110; - e_meta.data = 0b1111_1111_1111_1100; - assert_eq!(e_meta.status(), ErasureMetaStatus::CanRecover); - - e_meta.data = 0b1111_1111_1111_1000; - assert_eq!(e_meta.status(), ErasureMetaStatus::CanRecover); + assert_eq!(e_meta.status(), ErasureMetaStatus::DataFull); + } } #[test] fn test_meta_data_present() { let mut e_meta = ErasureMeta::default(); + e_meta.set_data_multi(0..NUM_DATA as u64, true); for i in 0..NUM_DATA as u64 { - e_meta.set_data_present(i, true); assert_eq!(e_meta.is_data_present(i), true); } for i in NUM_DATA as u64..2 * NUM_DATA as u64 { @@ -306,12 +335,13 @@ fn test_meta_data_present() { } e_meta.set_index = ErasureMeta::set_index_for((NUM_DATA * 23) as u64); + let start_idx = e_meta.start_index(); + e_meta.set_data_multi(start_idx..start_idx + NUM_DATA as u64, true); - for i in (NUM_DATA * 23) as u64..(NUM_DATA * 24) as u64 { - e_meta.set_data_present(i, true); + for i in start_idx..start_idx + NUM_DATA as u64 { assert_eq!(e_meta.is_data_present(i), true); } - for i in (NUM_DATA * 22) as u64..(NUM_DATA * 23) as u64 { + for i in start_idx - NUM_DATA as u64..start_idx { assert_eq!(e_meta.is_data_present(i), false); } } diff --git a/core/src/erasure.rs b/core/src/erasure.rs index 473e73e4f6..baa963b69d 100644 --- a/core/src/erasure.rs +++ b/core/src/erasure.rs @@ -49,9 +49,9 @@ use reed_solomon_erasure::ReedSolomon; //TODO(sakridge) pick these values /// Number of data blobs -pub const NUM_DATA: usize = 16; +pub const NUM_DATA: usize = 8; /// Number of coding blobs; also the maximum number that can go missing. -pub const NUM_CODING: usize = 4; +pub const NUM_CODING: usize = 8; /// Total number of blobs in an erasure set; includes data and coding blobs pub const ERASURE_SET_SIZE: usize = NUM_DATA + NUM_CODING;