diff --git a/core/benches/blocktree.rs b/core/benches/blocktree.rs index 05a304e40c..1496be867b 100644 --- a/core/benches/blocktree.rs +++ b/core/benches/blocktree.rs @@ -18,7 +18,7 @@ fn bench_write_shreds(bench: &mut Bencher, entries: Vec, ledger_path: &Pa let blocktree = Blocktree::open(ledger_path).expect("Expected to be able to open database ledger"); bench.iter(move || { - let shreds = entries_to_test_shreds(entries.clone(), 0, 0, true); + let shreds = entries_to_test_shreds(entries.clone(), 0, 0, true, 0); blocktree.insert_shreds(shreds, None, false).unwrap(); }); @@ -36,7 +36,7 @@ fn setup_read_bench( let entries = create_ticks(num_large_shreds * 4 + num_small_shreds * 2, Hash::default()); // Convert the entries to shreds, write the shreds to the ledger - let shreds = entries_to_test_shreds(entries, slot, slot.saturating_sub(1), true); + let shreds = entries_to_test_shreds(entries, slot, slot.saturating_sub(1), true, 0); blocktree .insert_shreds(shreds, None, false) .expect("Expectd successful insertion of shreds into ledger"); @@ -129,7 +129,7 @@ fn bench_insert_data_shred_small(bench: &mut Bencher) { let num_entries = 32 * 1024; let entries = create_ticks(num_entries, Hash::default()); bench.iter(move || { - let shreds = entries_to_test_shreds(entries.clone(), 0, 0, true); + let shreds = entries_to_test_shreds(entries.clone(), 0, 0, true, 0); blocktree.insert_shreds(shreds, None, false).unwrap(); }); Blocktree::destroy(&ledger_path).expect("Expected successful database destruction"); @@ -144,7 +144,7 @@ fn bench_insert_data_shred_big(bench: &mut Bencher) { let num_entries = 32 * 1024; let entries = create_ticks(num_entries, Hash::default()); bench.iter(move || { - let shreds = entries_to_test_shreds(entries.clone(), 0, 0, true); + let shreds = entries_to_test_shreds(entries.clone(), 0, 0, true, 0); blocktree.insert_shreds(shreds, None, false).unwrap(); }); Blocktree::destroy(&ledger_path).expect("Expected successful database destruction"); diff --git a/core/benches/shredder.rs b/core/benches/shredder.rs index 081373fc2f..2ecd0bbd8d 100644 --- a/core/benches/shredder.rs +++ b/core/benches/shredder.rs @@ -35,7 +35,7 @@ fn bench_shredder_ticks(bencher: &mut Bencher) { let num_ticks = max_ticks_per_n_shreds(1) * num_shreds as u64; let entries = create_ticks(num_ticks, Hash::default()); bencher.iter(|| { - let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0).unwrap(); + let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0, 0).unwrap(); shredder.entries_to_shreds(&entries, true, 0); }) } @@ -50,7 +50,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) { let entries = make_large_unchained_entries(txs_per_entry, num_entries); // 1Mb bencher.iter(|| { - let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0).unwrap(); + let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0, 0).unwrap(); shredder.entries_to_shreds(&entries, true, 0); }) } @@ -63,7 +63,7 @@ fn bench_deshredder(bencher: &mut Bencher) { let num_shreds = ((10000 * 1000) + (shred_size - 1)) / shred_size; let num_ticks = max_ticks_per_n_shreds(1) * num_shreds as u64; let entries = create_ticks(num_ticks, Hash::default()); - let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp, 0).unwrap(); + let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp, 0, 0).unwrap(); let data_shreds = shredder.entries_to_shreds(&entries, true, 0).0; bencher.iter(|| { let raw = &mut Shredder::deshred(&data_shreds).unwrap(); @@ -75,7 +75,7 @@ fn bench_deshredder(bencher: &mut Bencher) { fn bench_deserialize_hdr(bencher: &mut Bencher) { let data = vec![0; SIZE_OF_DATA_SHRED_PAYLOAD]; - let shred = Shred::new_from_data(2, 1, 1, Some(&data), true, true, 0); + let shred = Shred::new_from_data(2, 1, 1, Some(&data), true, true, 0, 0); bencher.iter(|| { let payload = shred.payload.clone(); diff --git a/core/src/blockstream_service.rs b/core/src/blockstream_service.rs index 69530faaf5..aa36d14334 100644 --- a/core/src/blockstream_service.rs +++ b/core/src/blockstream_service.rs @@ -158,6 +158,7 @@ mod test { true, &Arc::new(Keypair::new()), entries, + 0, ) .unwrap(); diff --git a/core/src/broadcast_stage.rs b/core/src/broadcast_stage.rs index cdfd535ad4..3c4a09e61a 100644 --- a/core/src/broadcast_stage.rs +++ b/core/src/broadcast_stage.rs @@ -43,6 +43,7 @@ impl BroadcastStageType { receiver: Receiver, exit_sender: &Arc, blocktree: &Arc, + shred_version: u16, ) -> BroadcastStage { match self { BroadcastStageType::Standard => { @@ -53,7 +54,7 @@ impl BroadcastStageType { receiver, exit_sender, blocktree, - StandardBroadcastRun::new(keypair), + StandardBroadcastRun::new(keypair, shred_version), ) } @@ -63,7 +64,7 @@ impl BroadcastStageType { receiver, exit_sender, blocktree, - FailEntryVerificationBroadcastRun::new(), + FailEntryVerificationBroadcastRun::new(shred_version), ), BroadcastStageType::BroadcastFakeBlobs => BroadcastStage::new( @@ -72,7 +73,7 @@ impl BroadcastStageType { receiver, exit_sender, blocktree, - BroadcastFakeBlobsRun::new(0), + BroadcastFakeBlobsRun::new(0, shred_version), ), } } @@ -246,7 +247,7 @@ mod test { entry_receiver, &exit_sender, &blocktree, - StandardBroadcastRun::new(leader_keypair), + StandardBroadcastRun::new(leader_keypair, 0), ); MockBroadcastStage { diff --git a/core/src/broadcast_stage/broadcast_fake_blobs_run.rs b/core/src/broadcast_stage/broadcast_fake_blobs_run.rs index 7ce057cca4..4169f14f16 100644 --- a/core/src/broadcast_stage/broadcast_fake_blobs_run.rs +++ b/core/src/broadcast_stage/broadcast_fake_blobs_run.rs @@ -6,13 +6,15 @@ use solana_sdk::hash::Hash; pub(super) struct BroadcastFakeBlobsRun { last_blockhash: Hash, partition: usize, + shred_version: u16, } impl BroadcastFakeBlobsRun { - pub(super) fn new(partition: usize) -> Self { + pub(super) fn new(partition: usize, shred_version: u16) -> Self { Self { last_blockhash: Hash::default(), partition, + shred_version, } } } @@ -45,6 +47,7 @@ impl BroadcastRun for BroadcastFakeBlobsRun { RECOMMENDED_FEC_RATE, keypair.clone(), (bank.tick_height() % bank.ticks_per_slot()) as u8, + self.shred_version, ) .expect("Expected to create a new shredder"); diff --git a/core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs b/core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs index b9931cda91..e805898e40 100644 --- a/core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs +++ b/core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs @@ -2,11 +2,13 @@ use super::*; use solana_ledger::shred::{Shredder, RECOMMENDED_FEC_RATE}; use solana_sdk::hash::Hash; -pub(super) struct FailEntryVerificationBroadcastRun {} +pub(super) struct FailEntryVerificationBroadcastRun { + shred_version: u16, +} impl FailEntryVerificationBroadcastRun { - pub(super) fn new() -> Self { - Self {} + pub(super) fn new(shred_version: u16) -> Self { + Self { shred_version } } } @@ -43,6 +45,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun { RECOMMENDED_FEC_RATE, keypair.clone(), (bank.tick_height() % bank.ticks_per_slot()) as u8, + self.shred_version, ) .expect("Expected to create a new shredder"); diff --git a/core/src/broadcast_stage/standard_broadcast_run.rs b/core/src/broadcast_stage/standard_broadcast_run.rs index 7faa7b7499..e06290dfbb 100644 --- a/core/src/broadcast_stage/standard_broadcast_run.rs +++ b/core/src/broadcast_stage/standard_broadcast_run.rs @@ -35,16 +35,18 @@ pub(super) struct StandardBroadcastRun { current_slot_and_parent: Option<(u64, u64)>, slot_broadcast_start: Option, keypair: Arc, + shred_version: u16, } impl StandardBroadcastRun { - pub(super) fn new(keypair: Arc) -> Self { + pub(super) fn new(keypair: Arc, shred_version: u16) -> Self { Self { stats: BroadcastStats::default(), unfinished_slot: None, current_slot_and_parent: None, slot_broadcast_start: None, keypair, + shred_version, } } @@ -63,6 +65,7 @@ impl StandardBroadcastRun { true, true, max_ticks_in_slot & SHRED_TICK_REFERENCE_MASK, + self.shred_version, )) } else { None @@ -93,6 +96,7 @@ impl StandardBroadcastRun { RECOMMENDED_FEC_RATE, self.keypair.clone(), reference_tick, + self.shred_version, ) .expect("Expected to create a new shredder"); @@ -346,7 +350,7 @@ mod test { #[test] fn test_interrupted_slot_last_shred() { let keypair = Arc::new(Keypair::new()); - let mut run = StandardBroadcastRun::new(keypair.clone()); + let mut run = StandardBroadcastRun::new(keypair.clone(), 0); // Set up the slot to be interrupted let next_shred_index = 10; @@ -392,7 +396,7 @@ mod test { }; // Step 1: Make an incomplete transmission for slot 0 - let mut standard_broadcast_run = StandardBroadcastRun::new(leader_keypair.clone()); + let mut standard_broadcast_run = StandardBroadcastRun::new(leader_keypair.clone(), 0); standard_broadcast_run .process_receive_results(&cluster_info, &socket, &blocktree, receive_results) .unwrap(); @@ -468,7 +472,7 @@ mod test { last_tick_height: ticks.len() as u64, }; - let mut standard_broadcast_run = StandardBroadcastRun::new(leader_keypair); + let mut standard_broadcast_run = StandardBroadcastRun::new(leader_keypair, 0); standard_broadcast_run .process_receive_results(&cluster_info, &socket, &blocktree, receive_results) .unwrap(); diff --git a/core/src/chacha.rs b/core/src/chacha.rs index af0016d6e6..0249c7a102 100644 --- a/core/src/chacha.rs +++ b/core/src/chacha.rs @@ -149,6 +149,7 @@ mod tests { true, &Arc::new(keypair), entries, + 0, ) .unwrap(); @@ -164,8 +165,8 @@ mod tests { let mut hasher = Hasher::default(); hasher.hash(&buf[..size]); - // golden needs to be updated if blob stuff changes.... - let golden: Hash = "HLzH7Nrh4q2K5WTh3e9vPNFZ1QVYhVDRMN9u5v51GqpJ" + // golden needs to be updated if blob structure changes.... + let golden: Hash = "9K6NR4cazo7Jzk2CpyXmNaZMGqvfXG83JzyJipkoHare" .parse() .unwrap(); diff --git a/core/src/chacha_cuda.rs b/core/src/chacha_cuda.rs index 39ec67994e..5949e2defa 100644 --- a/core/src/chacha_cuda.rs +++ b/core/src/chacha_cuda.rs @@ -147,6 +147,7 @@ mod tests { true, &Arc::new(Keypair::new()), entries, + 0, ) .unwrap(); @@ -208,6 +209,7 @@ mod tests { true, &Arc::new(Keypair::new()), entries, + 0, ) .unwrap(); diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index a9d27c932d..d80e0658ba 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -1087,7 +1087,7 @@ mod test { ), // should cause AccountNotFound error ], ); - entries_to_test_shreds(vec![entry], slot, slot.saturating_sub(1), false) + entries_to_test_shreds(vec![entry], slot, slot.saturating_sub(1), false, 0) }); assert_matches!( @@ -1112,7 +1112,7 @@ mod test { *blockhash, )], ); - entries_to_test_shreds(vec![entry], slot, slot.saturating_sub(1), false) + entries_to_test_shreds(vec![entry], slot, slot.saturating_sub(1), false, 0) }); assert_matches!(res, Err(Error::BlobError(BlobError::VerificationFailed))); diff --git a/core/src/retransmit_stage.rs b/core/src/retransmit_stage.rs index 400f8182ec..10fa646398 100644 --- a/core/src/retransmit_stage.rs +++ b/core/src/retransmit_stage.rs @@ -214,6 +214,7 @@ impl RetransmitStage { completed_slots_receiver: CompletedSlotsReceiver, epoch_schedule: EpochSchedule, cfg: Option, + shred_version: u16, ) -> Self { let (retransmit_sender, retransmit_receiver) = channel(); @@ -252,6 +253,7 @@ impl RetransmitStage { &leader_schedule_cache, id, last_root, + shred_version, ); rv && is_connected }, diff --git a/core/src/tpu.rs b/core/src/tpu.rs index 1f9923df8a..e68e574231 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -38,6 +38,7 @@ impl Tpu { blocktree: &Arc, broadcast_type: &BroadcastStageType, exit: &Arc, + shred_version: u16, ) -> Self { let (packet_sender, packet_receiver) = channel(); let fetch_stage = FetchStage::new_with_sender( @@ -74,6 +75,7 @@ impl Tpu { entry_receiver, &exit, blocktree, + shred_version, ); Self { diff --git a/core/src/tvu.rs b/core/src/tvu.rs index 0d2dcedaed..aa6da9acbc 100644 --- a/core/src/tvu.rs +++ b/core/src/tvu.rs @@ -81,6 +81,7 @@ impl Tvu { completed_slots_receiver: CompletedSlotsReceiver, fork_confidence_cache: Arc>, cfg: Option, + shred_version: u16, ) -> Self where T: 'static + KeypairUtil + Sync + Send, @@ -127,6 +128,7 @@ impl Tvu { completed_slots_receiver, *bank_forks.read().unwrap().working_bank().epoch_schedule(), cfg, + shred_version, ); let (blockstream_slot_sender, blockstream_slot_receiver) = channel(); @@ -290,6 +292,7 @@ pub mod tests { completed_slots_receiver, fork_confidence_cache, None, + 0, ); exit.store(true, Ordering::Relaxed); tvu.join().unwrap(); diff --git a/core/src/validator.rs b/core/src/validator.rs index 0f65675b77..c930ff2238 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -37,6 +37,7 @@ use solana_sdk::{ timing::timestamp, }; +use solana_ledger::shred::Shred; use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, path::{Path, PathBuf}, @@ -184,6 +185,8 @@ impl Validator { let bank = bank_forks[bank_info.bank_slot].clone(); let bank_forks = Arc::new(RwLock::new(bank_forks)); let fork_confidence_cache = Arc::new(RwLock::new(ForkConfidenceCache::default())); + // The version used by shreds, derived from genesis + let shred_version = Shred::version_from_hash(&genesis_blockhash); let mut validator_exit = ValidatorExit::default(); let exit_ = exit.clone(); @@ -344,6 +347,7 @@ impl Validator { completed_slots_receiver, fork_confidence_cache, config.partition_cfg.clone(), + shred_version, ); if config.dev_sigverify_disabled { @@ -361,6 +365,7 @@ impl Validator { &blocktree, &config.broadcast_stage_type, &exit, + shred_version, ); datapoint_info!("validator-new", ("id", id.to_string(), String)); diff --git a/core/src/window_service.rs b/core/src/window_service.rs index 427852be50..16426eae1a 100644 --- a/core/src/window_service.rs +++ b/core/src/window_service.rs @@ -41,6 +41,7 @@ pub fn should_retransmit_and_persist( leader_schedule_cache: &Arc, my_pubkey: &Pubkey, root: u64, + shred_version: u16, ) -> bool { let slot_leader_pubkey = match bank { None => leader_schedule_cache.slot_leader_at(shred.slot(), None), @@ -56,6 +57,9 @@ pub fn should_retransmit_and_persist( } else if !shred.verify(&leader_id) { inc_new_counter_debug!("streamer-recv_window-invalid_signature", 1); false + } else if shred.version() != shred_version { + inc_new_counter_debug!("streamer-recv_window-incorrect_shred_version", 1); + false } else { true } @@ -309,7 +313,7 @@ mod test { parent: u64, keypair: &Arc, ) -> Vec { - let shredder = Shredder::new(slot, parent, 0.0, keypair.clone(), 0) + let shredder = Shredder::new(slot, parent, 0.0, keypair.clone(), 0, 0) .expect("Failed to create entry shredder"); shredder.entries_to_shreds(&entries, true, 0).0 } @@ -349,25 +353,30 @@ mod test { // with a Bank for slot 0, blob continues assert_eq!( - should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, 0,), + should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, 0, 0), true ); + // with the wrong shred_version, shred gets thrown out + assert_eq!( + should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, 0, 1), + false + ); // If it's a coding shred, test that slot >= root - let (common, coding) = Shredder::new_coding_shred_header(5, 5, 6, 6, 0); + let (common, coding) = Shredder::new_coding_shred_header(5, 5, 6, 6, 0, 0); let mut coding_shred = Shred::new_empty_from_header(common, DataShredHeader::default(), coding); Shredder::sign_shred(&leader_keypair, &mut coding_shred); assert_eq!( - should_retransmit_and_persist(&coding_shred, Some(bank.clone()), &cache, &me_id, 0), + should_retransmit_and_persist(&coding_shred, Some(bank.clone()), &cache, &me_id, 0, 0), true ); assert_eq!( - should_retransmit_and_persist(&coding_shred, Some(bank.clone()), &cache, &me_id, 5), + should_retransmit_and_persist(&coding_shred, Some(bank.clone()), &cache, &me_id, 5, 0), true ); assert_eq!( - should_retransmit_and_persist(&coding_shred, Some(bank.clone()), &cache, &me_id, 6), + should_retransmit_and_persist(&coding_shred, Some(bank.clone()), &cache, &me_id, 6, 0), false ); @@ -384,7 +393,8 @@ mod test { Some(wrong_bank.clone()), &wrong_cache, &me_id, - 0 + 0, + 0, ), false ); @@ -392,7 +402,7 @@ mod test { // with a Bank and no idea who leader is, blob gets thrown out shreds[0].set_slot(MINIMUM_SLOTS_PER_EPOCH as u64 * 3); assert_eq!( - should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, 0), + should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, 0, 0), false ); @@ -400,7 +410,7 @@ mod test { let slot = MINIMUM_SLOTS_PER_EPOCH as u64 * 3; let shreds = local_entries_to_shred(&[Entry::default()], slot, slot - 1, &leader_keypair); assert_eq!( - should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, slot), + should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, slot, 0), false ); @@ -409,13 +419,13 @@ mod test { let shreds = local_entries_to_shred(&[Entry::default()], slot + 1, slot - 1, &leader_keypair); assert_eq!( - should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, slot), + should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, slot, 0), false ); // if the blob came back from me, it doesn't continue, whether or not I have a bank assert_eq!( - should_retransmit_and_persist(&shreds[0], None, &cache, &me_id, 0), + should_retransmit_and_persist(&shreds[0], None, &cache, &me_id, 0, 0), false ); } diff --git a/ledger/src/blocktree.rs b/ledger/src/blocktree.rs index 9b123e8c18..43c6e4d57e 100644 --- a/ledger/src/blocktree.rs +++ b/ledger/src/blocktree.rs @@ -894,6 +894,8 @@ impl Blocktree { self.code_shred_cf.get_bytes((slot, index)) } + // Only used by tests + #[allow(clippy::too_many_arguments)] pub fn write_entries( &self, start_slot: u64, @@ -904,6 +906,7 @@ impl Blocktree { is_full_slot: bool, keypair: &Arc, entries: Vec, + version: u16, ) -> Result { assert!(num_ticks_in_start_slot < ticks_per_slot); let mut remaining_ticks_in_slot = ticks_per_slot - num_ticks_in_start_slot; @@ -917,8 +920,9 @@ impl Blocktree { }, |v| v, ); - let mut shredder = Shredder::new(current_slot, parent_slot, 0.0, keypair.clone(), 0) - .expect("Failed to create entry shredder"); + let mut shredder = + Shredder::new(current_slot, parent_slot, 0.0, keypair.clone(), 0, version) + .expect("Failed to create entry shredder"); let mut all_shreds = vec![]; let mut slot_entries = vec![]; // Find all the entries for start_slot @@ -946,6 +950,7 @@ impl Blocktree { 0.0, keypair.clone(), (ticks_per_slot - remaining_ticks_in_slot) as u8, + version, ) .expect("Failed to create entry shredder"); } @@ -1740,8 +1745,9 @@ pub fn create_new_ledger(ledger_path: &Path, genesis_block: &GenesisBlock) -> Re let entries = create_ticks(ticks_per_slot, genesis_block.hash()); let last_hash = entries.last().unwrap().hash; + let version = Shred::version_from_hash(&last_hash); - let shredder = Shredder::new(0, 0, 0.0, Arc::new(Keypair::new()), 0) + let shredder = Shredder::new(0, 0, 0.0, Arc::new(Keypair::new()), 0, version) .expect("Failed to create entry shredder"); let shreds = shredder.entries_to_shreds(&entries, true, 0).0; assert!(shreds.last().unwrap().last_in_slot()); @@ -1825,8 +1831,9 @@ pub fn entries_to_test_shreds( slot: u64, parent_slot: u64, is_full_slot: bool, + version: u16, ) -> Vec { - let shredder = Shredder::new(slot, parent_slot, 0.0, Arc::new(Keypair::new()), 0) + let shredder = Shredder::new(slot, parent_slot, 0.0, Arc::new(Keypair::new()), 0, version) .expect("Failed to create entry shredder"); shredder.entries_to_shreds(&entries, is_full_slot, 0).0 @@ -1838,7 +1845,7 @@ pub fn make_slot_entries( num_entries: u64, ) -> (Vec, Vec) { let entries = create_ticks(num_entries, Hash::default()); - let shreds = entries_to_test_shreds(entries.clone(), slot, parent_slot, true); + let shreds = entries_to_test_shreds(entries.clone(), slot, parent_slot, true, 0); (shreds, entries) } @@ -2006,6 +2013,7 @@ pub mod tests { true, &Arc::new(Keypair::new()), new_ticks.clone(), + 0, ) .unwrap() as u64; shreds_per_slot.push(num_shreds); @@ -2326,12 +2334,12 @@ pub mod tests { { let blocktree = Blocktree::open(&blocktree_path).unwrap(); let entries = create_ticks(8, Hash::default()); - let shreds = entries_to_test_shreds(entries[0..4].to_vec(), 1, 0, false); + let shreds = entries_to_test_shreds(entries[0..4].to_vec(), 1, 0, false, 0); blocktree .insert_shreds(shreds, None, false) .expect("Expected successful write of shreds"); - let mut shreds1 = entries_to_test_shreds(entries[4..].to_vec(), 1, 0, false); + let mut shreds1 = entries_to_test_shreds(entries[4..].to_vec(), 1, 0, false, 0); for (i, b) in shreds1.iter_mut().enumerate() { b.set_index(8 + i as u32); } @@ -2363,7 +2371,7 @@ pub mod tests { let entries = create_ticks(slot + 1, Hash::default()); let last_entry = entries.last().unwrap().clone(); let mut shreds = - entries_to_test_shreds(entries, slot, slot.saturating_sub(1), false); + entries_to_test_shreds(entries, slot, slot.saturating_sub(1), false, 0); for b in shreds.iter_mut() { b.set_index(index); b.set_slot(slot as u64); @@ -2400,7 +2408,7 @@ pub mod tests { for slot in 0..num_slots { let entries = create_ticks(entries_per_slot, Hash::default()); let shreds = - entries_to_test_shreds(entries.clone(), slot, slot.saturating_sub(1), false); + entries_to_test_shreds(entries.clone(), slot, slot.saturating_sub(1), false, 0); assert!(shreds.len() as u64 >= shreds_per_slot); blocktree .insert_shreds(shreds, None, false) @@ -2495,7 +2503,7 @@ pub mod tests { assert_eq!(blocktree.get_slot_entries(0, 0, None).unwrap(), vec![]); - let duplicate_shreds = entries_to_test_shreds(original_entries.clone(), 0, 0, true); + let duplicate_shreds = entries_to_test_shreds(original_entries.clone(), 0, 0, true, 0); let num_shreds = duplicate_shreds.len() as u64; blocktree .insert_shreds(duplicate_shreds, None, false) @@ -3202,7 +3210,7 @@ pub mod tests { // Create enough entries to ensure there are at least two shreds created let num_entries = max_ticks_per_n_shreds(1) + 1; let entries = create_ticks(num_entries, Hash::default()); - let mut shreds = entries_to_test_shreds(entries, slot, 0, true); + let mut shreds = entries_to_test_shreds(entries, slot, 0, true, 0); let num_shreds = shreds.len(); assert!(num_shreds > 1); for (i, s) in shreds.iter_mut().enumerate() { @@ -3289,7 +3297,9 @@ pub mod tests { // Write entries let gap: u64 = 10; let shreds: Vec<_> = (0..64) - .map(|i| Shred::new_from_data(slot, (i * gap) as u32, 0, None, false, false, i as u8)) + .map(|i| { + Shred::new_from_data(slot, (i * gap) as u32, 0, None, false, false, i as u8, 0) + }) .collect(); blocktree.insert_shreds(shreds, None, false).unwrap(); @@ -3323,7 +3333,7 @@ pub mod tests { assert_eq!(blocktree.find_missing_data_indexes(slot, 0, 1, 2, 0), empty); let entries = create_ticks(100, Hash::default()); - let mut shreds = entries_to_test_shreds(entries, slot, 0, true); + let mut shreds = entries_to_test_shreds(entries, slot, 0, true, 0); assert!(shreds.len() > 2); shreds.drain(2..); @@ -3365,7 +3375,7 @@ pub mod tests { // Write entries let num_entries = 10; let entries = create_ticks(num_entries, Hash::default()); - let shreds = entries_to_test_shreds(entries, slot, 0, true); + let shreds = entries_to_test_shreds(entries, slot, 0, true, 0); let num_shreds = shreds.len(); blocktree.insert_shreds(shreds, None, false).unwrap(); @@ -3479,7 +3489,7 @@ pub mod tests { let last_root = RwLock::new(0); let slot = 1; - let (mut shred, coding) = Shredder::new_coding_shred_header(slot, 11, 11, 11, 10); + let (mut shred, coding) = Shredder::new_coding_shred_header(slot, 11, 11, 11, 10, 0); let coding_shred = Shred::new_empty_from_header( shred.clone(), DataShredHeader::default(), @@ -3883,7 +3893,7 @@ pub mod tests { let num_ticks = 8; let entries = create_ticks(num_ticks, Hash::default()); let slot = 1; - let shreds = entries_to_test_shreds(entries, slot, 0, false); + let shreds = entries_to_test_shreds(entries, slot, 0, false, 0); let next_shred_index = shreds.len(); blocktree .insert_shreds(shreds, None, false) @@ -3902,6 +3912,7 @@ pub mod tests { true, true, 0, + 0, )]; // With the corruption, nothing should be returned, even though an diff --git a/ledger/src/blocktree_processor.rs b/ledger/src/blocktree_processor.rs index ede1a49fab..c17ab082e7 100644 --- a/ledger/src/blocktree_processor.rs +++ b/ledger/src/blocktree_processor.rs @@ -494,6 +494,7 @@ pub fn fill_blocktree_slot_with_ticks( true, &Arc::new(Keypair::new()), entries, + 0, ) .unwrap(); @@ -568,6 +569,7 @@ pub mod tests { false, &Arc::new(Keypair::new()), entries, + 0, ) .expect("Expected to write shredded entries to blocktree"); } @@ -927,6 +929,7 @@ pub mod tests { true, &Arc::new(Keypair::new()), entries, + 0, ) .unwrap(); let opts = ProcessOptions { @@ -1033,6 +1036,7 @@ pub mod tests { true, &Arc::new(Keypair::new()), entries, + 0, ) .unwrap(); diff --git a/ledger/src/shred.rs b/ledger/src/shred.rs index 03aa80fec1..faa014b686 100644 --- a/ledger/src/shred.rs +++ b/ledger/src/shred.rs @@ -21,7 +21,7 @@ use std::{sync::Arc, time::Instant}; /// The following constants are computed by hand, and hardcoded. /// `test_shred_constants` ensures that the values are correct. /// Constants are used over lazy_static for performance reasons. -pub const SIZE_OF_COMMON_SHRED_HEADER: usize = 77; +pub const SIZE_OF_COMMON_SHRED_HEADER: usize = 79; pub const SIZE_OF_DATA_SHRED_HEADER: usize = 3; pub const SIZE_OF_CODING_SHRED_HEADER: usize = 6; pub const SIZE_OF_SIGNATURE: usize = 64; @@ -79,6 +79,7 @@ pub struct ShredCommonHeader { pub shred_type: ShredType, pub slot: u64, pub index: u32, + pub version: u16, } /// The data shred header has parent offset and flags @@ -136,15 +137,20 @@ impl Shred { is_last_data: bool, is_last_in_slot: bool, reference_tick: u8, + version: u16, ) -> Self { let mut payload = vec![0; PACKET_DATA_SIZE]; - let mut common_header = ShredCommonHeader::default(); - common_header.slot = slot; - common_header.index = index; + let common_header = ShredCommonHeader { + slot, + index, + version, + ..ShredCommonHeader::default() + }; - let mut data_header = DataShredHeader::default(); - data_header.parent_offset = parent_offset; - data_header.flags = reference_tick.min(SHRED_TICK_REFERENCE_MASK); + let mut data_header = DataShredHeader { + parent_offset, + flags: reference_tick.min(SHRED_TICK_REFERENCE_MASK), + }; if is_last_data { data_header.flags |= DATA_COMPLETE_SHRED @@ -275,6 +281,10 @@ impl Shred { self.common_header.index } + pub fn version(&self) -> u16 { + self.common_header.version + } + /// This is not a safe function. It only changes the meta information. /// Use this only for test code which doesn't care about actual shred pub fn set_index(&mut self, index: u32) { @@ -347,12 +357,26 @@ impl Shred { self.signature() .verify(pubkey.as_ref(), &self.payload[SIZE_OF_SIGNATURE..]) } + + pub fn version_from_hash(hash: &Hash) -> u16 { + let hash = hash.as_ref(); + let mut accum = [0u8; 2]; + hash.chunks(2).for_each(|seed| { + accum + .iter_mut() + .zip(seed) + .for_each(|(accum, seed)| *accum ^= *seed) + }); + // convert accum into a u16 + ((accum[0] as u16) << 8) | accum[1] as u16 + } } #[derive(Debug)] pub struct Shredder { slot: u64, parent_slot: u64, + version: u16, fec_rate: f32, keypair: Arc, pub signing_coding_time: u128, @@ -366,6 +390,7 @@ impl Shredder { fec_rate: f32, keypair: Arc, reference_tick: u8, + version: u16, ) -> Result { if fec_rate > 1.0 || fec_rate < 0.0 { Err(ShredError::InvalidFecRate(fec_rate)) @@ -379,6 +404,7 @@ impl Shredder { keypair, signing_coding_time: 0, reference_tick, + version, }) } } @@ -423,6 +449,7 @@ impl Shredder { is_last_data, is_last_in_slot, self.reference_tick, + self.version, ); Shredder::sign_shred(&self.keypair, &mut shred); @@ -438,7 +465,12 @@ impl Shredder { data_shreds .par_chunks(MAX_DATA_SHREDS_PER_FEC_BLOCK as usize) .flat_map(|shred_data_batch| { - Shredder::generate_coding_shreds(self.slot, self.fec_rate, shred_data_batch) + Shredder::generate_coding_shreds( + self.slot, + self.fec_rate, + shred_data_batch, + self.version, + ) }) .collect() }) @@ -480,11 +512,15 @@ impl Shredder { num_data: usize, num_code: usize, position: usize, + version: u16, ) -> (ShredCommonHeader, CodingShredHeader) { - let mut header = ShredCommonHeader::default(); - header.shred_type = ShredType(CODING_SHRED); - header.index = index; - header.slot = slot; + let header = ShredCommonHeader { + shred_type: ShredType(CODING_SHRED), + index, + slot, + version, + ..ShredCommonHeader::default() + }; ( header, CodingShredHeader { @@ -500,6 +536,7 @@ impl Shredder { slot: u64, fec_rate: f32, data_shred_batch: &[Shred], + version: u16, ) -> Vec { assert!(!data_shred_batch.is_empty()); if fec_rate != 0.0 { @@ -526,6 +563,7 @@ impl Shredder { num_data, num_coding, i, + version, ); let shred = Shred::new_empty_from_header(header, DataShredHeader::default(), coding_header); @@ -555,6 +593,7 @@ impl Shredder { num_data, num_coding, i, + version, ); Shred { common_header, @@ -758,6 +797,7 @@ pub mod tests { use super::*; use bincode::serialized_size; use matches::assert_matches; + use solana_sdk::hash::hash; use solana_sdk::system_transaction; use std::collections::HashSet; use std::convert::TryInto; @@ -825,7 +865,7 @@ pub mod tests { // Test that parent cannot be > current slot assert_matches!( - Shredder::new(slot, slot + 1, 1.00, keypair.clone(), 0), + Shredder::new(slot, slot + 1, 1.00, keypair.clone(), 0, 0), Err(ShredError::SlotTooLow { slot: _, parent_slot: _, @@ -833,7 +873,7 @@ pub mod tests { ); // Test that slot - parent cannot be > u16 MAX assert_matches!( - Shredder::new(slot, slot - 1 - 0xffff, 1.00, keypair.clone(), 0), + Shredder::new(slot, slot - 1 - 0xffff, 1.00, keypair.clone(), 0, 0), Err(ShredError::SlotTooLow { slot: _, parent_slot: _, @@ -842,7 +882,7 @@ pub mod tests { let fec_rate = 0.25; let parent_slot = slot - 5; - let shredder = Shredder::new(slot, parent_slot, fec_rate, keypair.clone(), 0) + let shredder = Shredder::new(slot, parent_slot, fec_rate, keypair.clone(), 0, 0) .expect("Failed in creating shredder"); let entries: Vec<_> = (0..5) @@ -917,7 +957,7 @@ pub mod tests { let slot = 1; let parent_slot = 0; - let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), 0) + let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), 0, 0) .expect("Failed in creating shredder"); let entries: Vec<_> = (0..5) @@ -943,7 +983,7 @@ pub mod tests { let slot = 1; let parent_slot = 0; - let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), 5) + let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), 5, 0) .expect("Failed in creating shredder"); let entries: Vec<_> = (0..5) @@ -973,7 +1013,7 @@ pub mod tests { let slot = 1; let parent_slot = 0; - let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), u8::max_value()) + let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), u8::max_value(), 0) .expect("Failed in creating shredder"); let entries: Vec<_> = (0..5) @@ -1010,11 +1050,11 @@ pub mod tests { let slot = 0x123456789abcdef0; // Test that FEC rate cannot be > 1.0 assert_matches!( - Shredder::new(slot, slot - 5, 1.001, keypair.clone(), 0), + Shredder::new(slot, slot - 5, 1.001, keypair.clone(), 0, 0), Err(ShredError::InvalidFecRate(_)) ); - let shredder = Shredder::new(0x123456789abcdef0, slot - 5, 1.0, keypair.clone(), 0) + let shredder = Shredder::new(0x123456789abcdef0, slot - 5, 1.0, keypair.clone(), 0, 0) .expect("Failed in creating shredder"); // Create enough entries to make > 1 shred @@ -1056,7 +1096,7 @@ pub mod tests { fn test_recovery_and_reassembly() { let keypair = Arc::new(Keypair::new()); let slot = 0x123456789abcdef0; - let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone(), 0) + let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone(), 0, 0) .expect("Failed in creating shredder"); let keypair0 = Keypair::new(); @@ -1302,7 +1342,7 @@ pub mod tests { fn test_multi_fec_block_coding() { let keypair = Arc::new(Keypair::new()); let slot = 0x123456789abcdef0; - let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone(), 0) + let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone(), 0, 0) .expect("Failed in creating shredder"); let num_fec_sets = 100; @@ -1385,4 +1425,54 @@ pub mod tests { let result = Shredder::deshred(&all_shreds[..]).unwrap(); assert_eq!(serialized_entries[..], result[..serialized_entries.len()]); } + + #[test] + fn test_shred_version() { + let keypair = Arc::new(Keypair::new()); + let hash = hash(Hash::default().as_ref()); + let version = Shred::version_from_hash(&hash); + assert_ne!(version, 0); + let shredder = + Shredder::new(0, 0, 1.0, keypair, 0, version).expect("Failed in creating shredder"); + + let entries: Vec<_> = (0..5) + .map(|_| { + let keypair0 = Keypair::new(); + let keypair1 = Keypair::new(); + let tx0 = + system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default()); + Entry::new(&Hash::default(), 1, vec![tx0]) + }) + .collect(); + + let (data_shreds, coding_shreds, _next_index) = + shredder.entries_to_shreds(&entries, true, 0); + assert!(!data_shreds + .iter() + .chain(coding_shreds.iter()) + .any(|s| s.version() != version)); + } + + #[test] + fn test_version_from_hash() { + let hash = [ + 0xa5u8, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, + 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, + 0xa5, 0xa5, 0x5a, 0x5a, + ]; + let version = Shred::version_from_hash(&Hash::new(&hash)); + assert_eq!(version, 0); + let hash = [ + 0xa5u8, 0xa5, 0x5a, 0x5a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + ]; + let version = Shred::version_from_hash(&Hash::new(&hash)); + assert_eq!(version, 0xffff); + let hash = [ + 0xa5u8, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + let version = Shred::version_from_hash(&Hash::new(&hash)); + assert_eq!(version, 0x5a5a); + } } diff --git a/ledger/tests/blocktree.rs b/ledger/tests/blocktree.rs index 7f359b0dce..427dd6990b 100644 --- a/ledger/tests/blocktree.rs +++ b/ledger/tests/blocktree.rs @@ -20,7 +20,7 @@ fn test_multiple_threads_insert_shred() { let threads: Vec<_> = (0..num_threads) .map(|i| { let entries = entry::create_ticks(1, Hash::default()); - let shreds = blocktree::entries_to_test_shreds(entries, i + 1, 0, false); + let shreds = blocktree::entries_to_test_shreds(entries, i + 1, 0, false, 0); let blocktree_ = blocktree.clone(); Builder::new() .name("blocktree-writer".to_string())