Add a version field to shreds (bp #7023) (#7025)

automerge

core/benches/blocktree.rs

@@ -18,7 +18,7 @@ fn bench_write_shreds(bench: &mut Bencher, entries: Vec<Entry>, ledger_path: &Pa
     let blocktree =
         Blocktree::open(ledger_path).expect("Expected to be able to open database ledger");
     bench.iter(move || {
-        let shreds = entries_to_test_shreds(entries.clone(), 0, 0, true);
+        let shreds = entries_to_test_shreds(entries.clone(), 0, 0, true, 0);
         blocktree.insert_shreds(shreds, None, false).unwrap();
     });
 
@@ -36,7 +36,7 @@ fn setup_read_bench(
     let entries = create_ticks(num_large_shreds * 4 + num_small_shreds * 2, Hash::default());
 
     // Convert the entries to shreds, write the shreds to the ledger
-    let shreds = entries_to_test_shreds(entries, slot, slot.saturating_sub(1), true);
+    let shreds = entries_to_test_shreds(entries, slot, slot.saturating_sub(1), true, 0);
     blocktree
         .insert_shreds(shreds, None, false)
         .expect("Expectd successful insertion of shreds into ledger");
@@ -129,7 +129,7 @@ fn bench_insert_data_shred_small(bench: &mut Bencher) {
     let num_entries = 32 * 1024;
     let entries = create_ticks(num_entries, Hash::default());
     bench.iter(move || {
-        let shreds = entries_to_test_shreds(entries.clone(), 0, 0, true);
+        let shreds = entries_to_test_shreds(entries.clone(), 0, 0, true, 0);
         blocktree.insert_shreds(shreds, None, false).unwrap();
     });
     Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");
@@ -144,7 +144,7 @@ fn bench_insert_data_shred_big(bench: &mut Bencher) {
     let num_entries = 32 * 1024;
     let entries = create_ticks(num_entries, Hash::default());
     bench.iter(move || {
-        let shreds = entries_to_test_shreds(entries.clone(), 0, 0, true);
+        let shreds = entries_to_test_shreds(entries.clone(), 0, 0, true, 0);
         blocktree.insert_shreds(shreds, None, false).unwrap();
     });
     Blocktree::destroy(&ledger_path).expect("Expected successful database destruction");

core/benches/shredder.rs

@@ -35,7 +35,7 @@ fn bench_shredder_ticks(bencher: &mut Bencher) {
     let num_ticks = max_ticks_per_n_shreds(1) * num_shreds as u64;
     let entries = create_ticks(num_ticks, Hash::default());
     bencher.iter(|| {
-        let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0).unwrap();
+        let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0, 0).unwrap();
         shredder.entries_to_shreds(&entries, true, 0);
     })
 }
@@ -50,7 +50,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
     let entries = make_large_unchained_entries(txs_per_entry, num_entries);
     // 1Mb
     bencher.iter(|| {
-        let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0).unwrap();
+        let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0, 0).unwrap();
         shredder.entries_to_shreds(&entries, true, 0);
     })
 }
@@ -63,7 +63,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
     let num_shreds = ((10000 * 1000) + (shred_size - 1)) / shred_size;
     let num_ticks = max_ticks_per_n_shreds(1) * num_shreds as u64;
     let entries = create_ticks(num_ticks, Hash::default());
-    let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp, 0).unwrap();
+    let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp, 0, 0).unwrap();
     let data_shreds = shredder.entries_to_shreds(&entries, true, 0).0;
     bencher.iter(|| {
         let raw = &mut Shredder::deshred(&data_shreds).unwrap();
@@ -75,7 +75,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
 fn bench_deserialize_hdr(bencher: &mut Bencher) {
     let data = vec![0; SIZE_OF_DATA_SHRED_PAYLOAD];
 
-    let shred = Shred::new_from_data(2, 1, 1, Some(&data), true, true, 0);
+    let shred = Shred::new_from_data(2, 1, 1, Some(&data), true, true, 0, 0);
 
     bencher.iter(|| {
         let payload = shred.payload.clone();

core/src/blockstream_service.rs

@@ -158,6 +158,7 @@ mod test {
                 true,
                 &Arc::new(Keypair::new()),
                 entries,
+                0,
             )
             .unwrap();
 

core/src/broadcast_stage.rs

@@ -43,6 +43,7 @@ impl BroadcastStageType {
         receiver: Receiver<WorkingBankEntry>,
         exit_sender: &Arc<AtomicBool>,
         blocktree: &Arc<Blocktree>,
+        shred_version: u16,
     ) -> BroadcastStage {
         match self {
             BroadcastStageType::Standard => {
@@ -53,7 +54,7 @@ impl BroadcastStageType {
                     receiver,
                     exit_sender,
                     blocktree,
-                    StandardBroadcastRun::new(keypair),
+                    StandardBroadcastRun::new(keypair, shred_version),
                 )
             }
 
@@ -63,7 +64,7 @@ impl BroadcastStageType {
                 receiver,
                 exit_sender,
                 blocktree,
-                FailEntryVerificationBroadcastRun::new(),
+                FailEntryVerificationBroadcastRun::new(shred_version),
             ),
 
             BroadcastStageType::BroadcastFakeBlobs => BroadcastStage::new(
@@ -72,7 +73,7 @@ impl BroadcastStageType {
                 receiver,
                 exit_sender,
                 blocktree,
-                BroadcastFakeBlobsRun::new(0),
+                BroadcastFakeBlobsRun::new(0, shred_version),
             ),
         }
     }
@@ -246,7 +247,7 @@ mod test {
             entry_receiver,
             &exit_sender,
             &blocktree,
-            StandardBroadcastRun::new(leader_keypair),
+            StandardBroadcastRun::new(leader_keypair, 0),
         );
 
         MockBroadcastStage {

core/src/broadcast_stage/broadcast_fake_blobs_run.rs

@@ -6,13 +6,15 @@ use solana_sdk::hash::Hash;
 pub(super) struct BroadcastFakeBlobsRun {
     last_blockhash: Hash,
     partition: usize,
+    shred_version: u16,
 }
 
 impl BroadcastFakeBlobsRun {
-    pub(super) fn new(partition: usize) -> Self {
+    pub(super) fn new(partition: usize, shred_version: u16) -> Self {
         Self {
             last_blockhash: Hash::default(),
             partition,
+            shred_version,
         }
     }
 }
@@ -45,6 +47,7 @@ impl BroadcastRun for BroadcastFakeBlobsRun {
             RECOMMENDED_FEC_RATE,
             keypair.clone(),
             (bank.tick_height() % bank.ticks_per_slot()) as u8,
+            self.shred_version,
         )
         .expect("Expected to create a new shredder");
 

core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs

@@ -2,11 +2,13 @@ use super::*;
 use solana_ledger::shred::{Shredder, RECOMMENDED_FEC_RATE};
 use solana_sdk::hash::Hash;
 
-pub(super) struct FailEntryVerificationBroadcastRun {}
+pub(super) struct FailEntryVerificationBroadcastRun {
+    shred_version: u16,
+}
 
 impl FailEntryVerificationBroadcastRun {
-    pub(super) fn new() -> Self {
+    pub(super) fn new(shred_version: u16) -> Self {
-        Self {}
+        Self { shred_version }
     }
 }
 
@@ -43,6 +45,7 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
             RECOMMENDED_FEC_RATE,
             keypair.clone(),
             (bank.tick_height() % bank.ticks_per_slot()) as u8,
+            self.shred_version,
         )
         .expect("Expected to create a new shredder");
 

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -35,16 +35,18 @@ pub(super) struct StandardBroadcastRun {
     current_slot_and_parent: Option<(u64, u64)>,
     slot_broadcast_start: Option<Instant>,
     keypair: Arc<Keypair>,
+    shred_version: u16,
 }
 
 impl StandardBroadcastRun {
-    pub(super) fn new(keypair: Arc<Keypair>) -> Self {
+    pub(super) fn new(keypair: Arc<Keypair>, shred_version: u16) -> Self {
         Self {
             stats: BroadcastStats::default(),
             unfinished_slot: None,
             current_slot_and_parent: None,
             slot_broadcast_start: None,
             keypair,
+            shred_version,
         }
     }
 
@@ -63,6 +65,7 @@ impl StandardBroadcastRun {
                         true,
                         true,
                         max_ticks_in_slot & SHRED_TICK_REFERENCE_MASK,
+                        self.shred_version,
                     ))
                 } else {
                     None
@@ -93,6 +96,7 @@ impl StandardBroadcastRun {
             RECOMMENDED_FEC_RATE,
             self.keypair.clone(),
             reference_tick,
+            self.shred_version,
         )
         .expect("Expected to create a new shredder");
 
@@ -346,7 +350,7 @@ mod test {
     #[test]
     fn test_interrupted_slot_last_shred() {
         let keypair = Arc::new(Keypair::new());
-        let mut run = StandardBroadcastRun::new(keypair.clone());
+        let mut run = StandardBroadcastRun::new(keypair.clone(), 0);
 
         // Set up the slot to be interrupted
         let next_shred_index = 10;
@@ -392,7 +396,7 @@ mod test {
         };
 
         // Step 1: Make an incomplete transmission for slot 0
-        let mut standard_broadcast_run = StandardBroadcastRun::new(leader_keypair.clone());
+        let mut standard_broadcast_run = StandardBroadcastRun::new(leader_keypair.clone(), 0);
         standard_broadcast_run
             .process_receive_results(&cluster_info, &socket, &blocktree, receive_results)
             .unwrap();
@@ -468,7 +472,7 @@ mod test {
             last_tick_height: ticks.len() as u64,
         };
 
-        let mut standard_broadcast_run = StandardBroadcastRun::new(leader_keypair);
+        let mut standard_broadcast_run = StandardBroadcastRun::new(leader_keypair, 0);
         standard_broadcast_run
             .process_receive_results(&cluster_info, &socket, &blocktree, receive_results)
             .unwrap();

core/src/chacha.rs

@@ -149,6 +149,7 @@ mod tests {
                 true,
                 &Arc::new(keypair),
                 entries,
+                0,
             )
             .unwrap();
 
@@ -164,8 +165,8 @@ mod tests {
         let mut hasher = Hasher::default();
         hasher.hash(&buf[..size]);
 
-        //  golden needs to be updated if blob stuff changes....
+        //  golden needs to be updated if blob structure changes....
-        let golden: Hash = "HLzH7Nrh4q2K5WTh3e9vPNFZ1QVYhVDRMN9u5v51GqpJ"
+        let golden: Hash = "9K6NR4cazo7Jzk2CpyXmNaZMGqvfXG83JzyJipkoHare"
             .parse()
             .unwrap();
 

core/src/chacha_cuda.rs

@@ -147,6 +147,7 @@ mod tests {
                 true,
                 &Arc::new(Keypair::new()),
                 entries,
+                0,
             )
             .unwrap();
 
@@ -208,6 +209,7 @@ mod tests {
                 true,
                 &Arc::new(Keypair::new()),
                 entries,
+                0,
             )
             .unwrap();
 

core/src/replay_stage.rs

@@ -1087,7 +1087,7 @@ mod test {
                     ), // should cause AccountNotFound error
                 ],
             );
-            entries_to_test_shreds(vec![entry], slot, slot.saturating_sub(1), false)
+            entries_to_test_shreds(vec![entry], slot, slot.saturating_sub(1), false, 0)
         });
 
         assert_matches!(
@@ -1112,7 +1112,7 @@ mod test {
                     *blockhash,
                 )],
             );
-            entries_to_test_shreds(vec![entry], slot, slot.saturating_sub(1), false)
+            entries_to_test_shreds(vec![entry], slot, slot.saturating_sub(1), false, 0)
         });
 
         assert_matches!(res, Err(Error::BlobError(BlobError::VerificationFailed)));

core/src/retransmit_stage.rs

@@ -214,6 +214,7 @@ impl RetransmitStage {
         completed_slots_receiver: CompletedSlotsReceiver,
         epoch_schedule: EpochSchedule,
         cfg: Option<PartitionCfg>,
+        shred_version: u16,
     ) -> Self {
         let (retransmit_sender, retransmit_receiver) = channel();
 
@@ -252,6 +253,7 @@ impl RetransmitStage {
                     &leader_schedule_cache,
                     id,
                     last_root,
+                    shred_version,
                 );
                 rv && is_connected
             },

core/src/tpu.rs

@@ -38,6 +38,7 @@ impl Tpu {
         blocktree: &Arc<Blocktree>,
         broadcast_type: &BroadcastStageType,
         exit: &Arc<AtomicBool>,
+        shred_version: u16,
     ) -> Self {
         let (packet_sender, packet_receiver) = channel();
         let fetch_stage = FetchStage::new_with_sender(
@@ -74,6 +75,7 @@ impl Tpu {
             entry_receiver,
             &exit,
             blocktree,
+            shred_version,
         );
 
         Self {

core/src/tvu.rs

@@ -81,6 +81,7 @@ impl Tvu {
         completed_slots_receiver: CompletedSlotsReceiver,
         fork_confidence_cache: Arc<RwLock<ForkConfidenceCache>>,
         cfg: Option<PartitionCfg>,
+        shred_version: u16,
     ) -> Self
     where
         T: 'static + KeypairUtil + Sync + Send,
@@ -127,6 +128,7 @@ impl Tvu {
             completed_slots_receiver,
             *bank_forks.read().unwrap().working_bank().epoch_schedule(),
             cfg,
+            shred_version,
         );
 
         let (blockstream_slot_sender, blockstream_slot_receiver) = channel();
@@ -290,6 +292,7 @@ pub mod tests {
             completed_slots_receiver,
             fork_confidence_cache,
             None,
+            0,
         );
         exit.store(true, Ordering::Relaxed);
         tvu.join().unwrap();

core/src/validator.rs

@@ -37,6 +37,7 @@ use solana_sdk::{
     timing::timestamp,
 };
 
+use solana_ledger::shred::Shred;
 use std::{
     net::{IpAddr, Ipv4Addr, SocketAddr},
     path::{Path, PathBuf},
@@ -184,6 +185,8 @@ impl Validator {
         let bank = bank_forks[bank_info.bank_slot].clone();
         let bank_forks = Arc::new(RwLock::new(bank_forks));
         let fork_confidence_cache = Arc::new(RwLock::new(ForkConfidenceCache::default()));
+        // The version used by shreds, derived from genesis
+        let shred_version = Shred::version_from_hash(&genesis_blockhash);
 
         let mut validator_exit = ValidatorExit::default();
         let exit_ = exit.clone();
@@ -344,6 +347,7 @@ impl Validator {
             completed_slots_receiver,
             fork_confidence_cache,
             config.partition_cfg.clone(),
+            shred_version,
         );
 
         if config.dev_sigverify_disabled {
@@ -361,6 +365,7 @@ impl Validator {
             &blocktree,
             &config.broadcast_stage_type,
             &exit,
+            shred_version,
         );
 
         datapoint_info!("validator-new", ("id", id.to_string(), String));

core/src/window_service.rs

@@ -41,6 +41,7 @@ pub fn should_retransmit_and_persist(
     leader_schedule_cache: &Arc<LeaderScheduleCache>,
     my_pubkey: &Pubkey,
     root: u64,
+    shred_version: u16,
 ) -> bool {
     let slot_leader_pubkey = match bank {
         None => leader_schedule_cache.slot_leader_at(shred.slot(), None),
@@ -56,6 +57,9 @@ pub fn should_retransmit_and_persist(
         } else if !shred.verify(&leader_id) {
             inc_new_counter_debug!("streamer-recv_window-invalid_signature", 1);
             false
+        } else if shred.version() != shred_version {
+            inc_new_counter_debug!("streamer-recv_window-incorrect_shred_version", 1);
+            false
         } else {
             true
         }
@@ -309,7 +313,7 @@ mod test {
         parent: u64,
         keypair: &Arc<Keypair>,
     ) -> Vec<Shred> {
-        let shredder = Shredder::new(slot, parent, 0.0, keypair.clone(), 0)
+        let shredder = Shredder::new(slot, parent, 0.0, keypair.clone(), 0, 0)
             .expect("Failed to create entry shredder");
         shredder.entries_to_shreds(&entries, true, 0).0
     }
@@ -349,25 +353,30 @@ mod test {
 
         // with a Bank for slot 0, blob continues
         assert_eq!(
-            should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, 0,),
+            should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, 0, 0),
             true
         );
+        // with the wrong shred_version, shred gets thrown out
+        assert_eq!(
+            should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, 0, 1),
+            false
+        );
 
         // If it's a coding shred, test that slot >= root
-        let (common, coding) = Shredder::new_coding_shred_header(5, 5, 6, 6, 0);
+        let (common, coding) = Shredder::new_coding_shred_header(5, 5, 6, 6, 0, 0);
         let mut coding_shred =
             Shred::new_empty_from_header(common, DataShredHeader::default(), coding);
         Shredder::sign_shred(&leader_keypair, &mut coding_shred);
         assert_eq!(
-            should_retransmit_and_persist(&coding_shred, Some(bank.clone()), &cache, &me_id, 0),
+            should_retransmit_and_persist(&coding_shred, Some(bank.clone()), &cache, &me_id, 0, 0),
             true
         );
         assert_eq!(
-            should_retransmit_and_persist(&coding_shred, Some(bank.clone()), &cache, &me_id, 5),
+            should_retransmit_and_persist(&coding_shred, Some(bank.clone()), &cache, &me_id, 5, 0),
             true
         );
         assert_eq!(
-            should_retransmit_and_persist(&coding_shred, Some(bank.clone()), &cache, &me_id, 6),
+            should_retransmit_and_persist(&coding_shred, Some(bank.clone()), &cache, &me_id, 6, 0),
             false
         );
 
@@ -384,7 +393,8 @@ mod test {
                 Some(wrong_bank.clone()),
                 &wrong_cache,
                 &me_id,
-                0
+                0,
+                0,
             ),
             false
         );
@@ -392,7 +402,7 @@ mod test {
         // with a Bank and no idea who leader is, blob gets thrown out
         shreds[0].set_slot(MINIMUM_SLOTS_PER_EPOCH as u64 * 3);
         assert_eq!(
-            should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, 0),
+            should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, 0, 0),
             false
         );
 
@@ -400,7 +410,7 @@ mod test {
         let slot = MINIMUM_SLOTS_PER_EPOCH as u64 * 3;
         let shreds = local_entries_to_shred(&[Entry::default()], slot, slot - 1, &leader_keypair);
         assert_eq!(
-            should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, slot),
+            should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, slot, 0),
             false
         );
 
@@ -409,13 +419,13 @@ mod test {
         let shreds =
             local_entries_to_shred(&[Entry::default()], slot + 1, slot - 1, &leader_keypair);
         assert_eq!(
-            should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, slot),
+            should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id, slot, 0),
             false
         );
 
         // if the blob came back from me, it doesn't continue, whether or not I have a bank
         assert_eq!(
-            should_retransmit_and_persist(&shreds[0], None, &cache, &me_id, 0),
+            should_retransmit_and_persist(&shreds[0], None, &cache, &me_id, 0, 0),
             false
         );
     }

ledger/src/blocktree.rs

@@ -894,6 +894,8 @@ impl Blocktree {
         self.code_shred_cf.get_bytes((slot, index))
     }
 
+    // Only used by tests
+    #[allow(clippy::too_many_arguments)]
     pub fn write_entries(
         &self,
         start_slot: u64,
@@ -904,6 +906,7 @@ impl Blocktree {
         is_full_slot: bool,
         keypair: &Arc<Keypair>,
         entries: Vec<Entry>,
+        version: u16,
     ) -> Result<usize> {
         assert!(num_ticks_in_start_slot < ticks_per_slot);
         let mut remaining_ticks_in_slot = ticks_per_slot - num_ticks_in_start_slot;
@@ -917,7 +920,8 @@ impl Blocktree {
             },
             |v| v,
         );
-        let mut shredder = Shredder::new(current_slot, parent_slot, 0.0, keypair.clone(), 0)
+        let mut shredder =
+            Shredder::new(current_slot, parent_slot, 0.0, keypair.clone(), 0, version)
                 .expect("Failed to create entry shredder");
         let mut all_shreds = vec![];
         let mut slot_entries = vec![];
@@ -946,6 +950,7 @@ impl Blocktree {
                     0.0,
                     keypair.clone(),
                     (ticks_per_slot - remaining_ticks_in_slot) as u8,
+                    version,
                 )
                 .expect("Failed to create entry shredder");
             }
@@ -1740,8 +1745,9 @@ pub fn create_new_ledger(ledger_path: &Path, genesis_block: &GenesisBlock) -> Re
 
     let entries = create_ticks(ticks_per_slot, genesis_block.hash());
     let last_hash = entries.last().unwrap().hash;
+    let version = Shred::version_from_hash(&last_hash);
 
-    let shredder = Shredder::new(0, 0, 0.0, Arc::new(Keypair::new()), 0)
+    let shredder = Shredder::new(0, 0, 0.0, Arc::new(Keypair::new()), 0, version)
         .expect("Failed to create entry shredder");
     let shreds = shredder.entries_to_shreds(&entries, true, 0).0;
     assert!(shreds.last().unwrap().last_in_slot());
@@ -1825,8 +1831,9 @@ pub fn entries_to_test_shreds(
     slot: u64,
     parent_slot: u64,
     is_full_slot: bool,
+    version: u16,
 ) -> Vec<Shred> {
-    let shredder = Shredder::new(slot, parent_slot, 0.0, Arc::new(Keypair::new()), 0)
+    let shredder = Shredder::new(slot, parent_slot, 0.0, Arc::new(Keypair::new()), 0, version)
         .expect("Failed to create entry shredder");
 
     shredder.entries_to_shreds(&entries, is_full_slot, 0).0
@@ -1838,7 +1845,7 @@ pub fn make_slot_entries(
     num_entries: u64,
 ) -> (Vec<Shred>, Vec<Entry>) {
     let entries = create_ticks(num_entries, Hash::default());
-    let shreds = entries_to_test_shreds(entries.clone(), slot, parent_slot, true);
+    let shreds = entries_to_test_shreds(entries.clone(), slot, parent_slot, true, 0);
     (shreds, entries)
 }
 
@@ -2006,6 +2013,7 @@ pub mod tests {
                         true,
                         &Arc::new(Keypair::new()),
                         new_ticks.clone(),
+                        0,
                     )
                     .unwrap() as u64;
                 shreds_per_slot.push(num_shreds);
@@ -2326,12 +2334,12 @@ pub mod tests {
         {
             let blocktree = Blocktree::open(&blocktree_path).unwrap();
             let entries = create_ticks(8, Hash::default());
-            let shreds = entries_to_test_shreds(entries[0..4].to_vec(), 1, 0, false);
+            let shreds = entries_to_test_shreds(entries[0..4].to_vec(), 1, 0, false, 0);
             blocktree
                 .insert_shreds(shreds, None, false)
                 .expect("Expected successful write of shreds");
 
-            let mut shreds1 = entries_to_test_shreds(entries[4..].to_vec(), 1, 0, false);
+            let mut shreds1 = entries_to_test_shreds(entries[4..].to_vec(), 1, 0, false, 0);
             for (i, b) in shreds1.iter_mut().enumerate() {
                 b.set_index(8 + i as u32);
             }
@@ -2363,7 +2371,7 @@ pub mod tests {
                 let entries = create_ticks(slot + 1, Hash::default());
                 let last_entry = entries.last().unwrap().clone();
                 let mut shreds =
-                    entries_to_test_shreds(entries, slot, slot.saturating_sub(1), false);
+                    entries_to_test_shreds(entries, slot, slot.saturating_sub(1), false, 0);
                 for b in shreds.iter_mut() {
                     b.set_index(index);
                     b.set_slot(slot as u64);
@@ -2400,7 +2408,7 @@ pub mod tests {
             for slot in 0..num_slots {
                 let entries = create_ticks(entries_per_slot, Hash::default());
                 let shreds =
-                    entries_to_test_shreds(entries.clone(), slot, slot.saturating_sub(1), false);
+                    entries_to_test_shreds(entries.clone(), slot, slot.saturating_sub(1), false, 0);
                 assert!(shreds.len() as u64 >= shreds_per_slot);
                 blocktree
                     .insert_shreds(shreds, None, false)
@@ -2495,7 +2503,7 @@ pub mod tests {
 
             assert_eq!(blocktree.get_slot_entries(0, 0, None).unwrap(), vec![]);
 
-            let duplicate_shreds = entries_to_test_shreds(original_entries.clone(), 0, 0, true);
+            let duplicate_shreds = entries_to_test_shreds(original_entries.clone(), 0, 0, true, 0);
             let num_shreds = duplicate_shreds.len() as u64;
             blocktree
                 .insert_shreds(duplicate_shreds, None, false)
@@ -3202,7 +3210,7 @@ pub mod tests {
         // Create enough entries to ensure there are at least two shreds created
         let num_entries = max_ticks_per_n_shreds(1) + 1;
         let entries = create_ticks(num_entries, Hash::default());
-        let mut shreds = entries_to_test_shreds(entries, slot, 0, true);
+        let mut shreds = entries_to_test_shreds(entries, slot, 0, true, 0);
         let num_shreds = shreds.len();
         assert!(num_shreds > 1);
         for (i, s) in shreds.iter_mut().enumerate() {
@@ -3289,7 +3297,9 @@ pub mod tests {
         // Write entries
         let gap: u64 = 10;
         let shreds: Vec<_> = (0..64)
-            .map(|i| Shred::new_from_data(slot, (i * gap) as u32, 0, None, false, false, i as u8))
+            .map(|i| {
+                Shred::new_from_data(slot, (i * gap) as u32, 0, None, false, false, i as u8, 0)
+            })
             .collect();
         blocktree.insert_shreds(shreds, None, false).unwrap();
 
@@ -3323,7 +3333,7 @@ pub mod tests {
         assert_eq!(blocktree.find_missing_data_indexes(slot, 0, 1, 2, 0), empty);
 
         let entries = create_ticks(100, Hash::default());
-        let mut shreds = entries_to_test_shreds(entries, slot, 0, true);
+        let mut shreds = entries_to_test_shreds(entries, slot, 0, true, 0);
         assert!(shreds.len() > 2);
         shreds.drain(2..);
 
@@ -3365,7 +3375,7 @@ pub mod tests {
         // Write entries
         let num_entries = 10;
         let entries = create_ticks(num_entries, Hash::default());
-        let shreds = entries_to_test_shreds(entries, slot, 0, true);
+        let shreds = entries_to_test_shreds(entries, slot, 0, true, 0);
         let num_shreds = shreds.len();
 
         blocktree.insert_shreds(shreds, None, false).unwrap();
@@ -3479,7 +3489,7 @@ pub mod tests {
             let last_root = RwLock::new(0);
 
             let slot = 1;
-            let (mut shred, coding) = Shredder::new_coding_shred_header(slot, 11, 11, 11, 10);
+            let (mut shred, coding) = Shredder::new_coding_shred_header(slot, 11, 11, 11, 10, 0);
             let coding_shred = Shred::new_empty_from_header(
                 shred.clone(),
                 DataShredHeader::default(),
@@ -3883,7 +3893,7 @@ pub mod tests {
             let num_ticks = 8;
             let entries = create_ticks(num_ticks, Hash::default());
             let slot = 1;
-            let shreds = entries_to_test_shreds(entries, slot, 0, false);
+            let shreds = entries_to_test_shreds(entries, slot, 0, false, 0);
             let next_shred_index = shreds.len();
             blocktree
                 .insert_shreds(shreds, None, false)
@@ -3902,6 +3912,7 @@ pub mod tests {
                 true,
                 true,
                 0,
+                0,
             )];
 
             // With the corruption, nothing should be returned, even though an

ledger/src/blocktree_processor.rs

@@ -494,6 +494,7 @@ pub fn fill_blocktree_slot_with_ticks(
             true,
             &Arc::new(Keypair::new()),
             entries,
+            0,
         )
         .unwrap();
 
@@ -568,6 +569,7 @@ pub mod tests {
                     false,
                     &Arc::new(Keypair::new()),
                     entries,
+                    0,
                 )
                 .expect("Expected to write shredded entries to blocktree");
         }
@@ -927,6 +929,7 @@ pub mod tests {
                 true,
                 &Arc::new(Keypair::new()),
                 entries,
+                0,
             )
             .unwrap();
         let opts = ProcessOptions {
@@ -1033,6 +1036,7 @@ pub mod tests {
                 true,
                 &Arc::new(Keypair::new()),
                 entries,
+                0,
             )
             .unwrap();
 

ledger/src/shred.rs

@@ -21,7 +21,7 @@ use std::{sync::Arc, time::Instant};
 /// The following constants are computed by hand, and hardcoded.
 /// `test_shred_constants` ensures that the values are correct.
 /// Constants are used over lazy_static for performance reasons.
-pub const SIZE_OF_COMMON_SHRED_HEADER: usize = 77;
+pub const SIZE_OF_COMMON_SHRED_HEADER: usize = 79;
 pub const SIZE_OF_DATA_SHRED_HEADER: usize = 3;
 pub const SIZE_OF_CODING_SHRED_HEADER: usize = 6;
 pub const SIZE_OF_SIGNATURE: usize = 64;
@@ -79,6 +79,7 @@ pub struct ShredCommonHeader {
     pub shred_type: ShredType,
     pub slot: u64,
     pub index: u32,
+    pub version: u16,
 }
 
 /// The data shred header has parent offset and flags
@@ -136,15 +137,20 @@ impl Shred {
         is_last_data: bool,
         is_last_in_slot: bool,
         reference_tick: u8,
+        version: u16,
     ) -> Self {
         let mut payload = vec![0; PACKET_DATA_SIZE];
-        let mut common_header = ShredCommonHeader::default();
+        let common_header = ShredCommonHeader {
-        common_header.slot = slot;
+            slot,
-        common_header.index = index;
+            index,
+            version,
+            ..ShredCommonHeader::default()
+        };
 
-        let mut data_header = DataShredHeader::default();
+        let mut data_header = DataShredHeader {
-        data_header.parent_offset = parent_offset;
+            parent_offset,
-        data_header.flags = reference_tick.min(SHRED_TICK_REFERENCE_MASK);
+            flags: reference_tick.min(SHRED_TICK_REFERENCE_MASK),
+        };
 
         if is_last_data {
             data_header.flags |= DATA_COMPLETE_SHRED
@@ -275,6 +281,10 @@ impl Shred {
         self.common_header.index
     }
 
+    pub fn version(&self) -> u16 {
+        self.common_header.version
+    }
+
     /// This is not a safe function. It only changes the meta information.
     /// Use this only for test code which doesn't care about actual shred
     pub fn set_index(&mut self, index: u32) {
@@ -347,12 +357,26 @@ impl Shred {
         self.signature()
             .verify(pubkey.as_ref(), &self.payload[SIZE_OF_SIGNATURE..])
     }
+
+    pub fn version_from_hash(hash: &Hash) -> u16 {
+        let hash = hash.as_ref();
+        let mut accum = [0u8; 2];
+        hash.chunks(2).for_each(|seed| {
+            accum
+                .iter_mut()
+                .zip(seed)
+                .for_each(|(accum, seed)| *accum ^= *seed)
+        });
+        // convert accum into a u16
+        ((accum[0] as u16) << 8) | accum[1] as u16
+    }
 }
 
 #[derive(Debug)]
 pub struct Shredder {
     slot: u64,
     parent_slot: u64,
+    version: u16,
     fec_rate: f32,
     keypair: Arc<Keypair>,
     pub signing_coding_time: u128,
@@ -366,6 +390,7 @@ impl Shredder {
         fec_rate: f32,
         keypair: Arc<Keypair>,
         reference_tick: u8,
+        version: u16,
     ) -> Result<Self> {
         if fec_rate > 1.0 || fec_rate < 0.0 {
             Err(ShredError::InvalidFecRate(fec_rate))
@@ -379,6 +404,7 @@ impl Shredder {
                 keypair,
                 signing_coding_time: 0,
                 reference_tick,
+                version,
             })
         }
     }
@@ -423,6 +449,7 @@ impl Shredder {
                             is_last_data,
                             is_last_in_slot,
                             self.reference_tick,
+                            self.version,
                         );
 
                         Shredder::sign_shred(&self.keypair, &mut shred);
@@ -438,7 +465,12 @@ impl Shredder {
                 data_shreds
                     .par_chunks(MAX_DATA_SHREDS_PER_FEC_BLOCK as usize)
                     .flat_map(|shred_data_batch| {
-                        Shredder::generate_coding_shreds(self.slot, self.fec_rate, shred_data_batch)
+                        Shredder::generate_coding_shreds(
+                            self.slot,
+                            self.fec_rate,
+                            shred_data_batch,
+                            self.version,
+                        )
                     })
                     .collect()
             })
@@ -480,11 +512,15 @@ impl Shredder {
         num_data: usize,
         num_code: usize,
         position: usize,
+        version: u16,
     ) -> (ShredCommonHeader, CodingShredHeader) {
-        let mut header = ShredCommonHeader::default();
+        let header = ShredCommonHeader {
-        header.shred_type = ShredType(CODING_SHRED);
+            shred_type: ShredType(CODING_SHRED),
-        header.index = index;
+            index,
-        header.slot = slot;
+            slot,
+            version,
+            ..ShredCommonHeader::default()
+        };
         (
             header,
             CodingShredHeader {
@@ -500,6 +536,7 @@ impl Shredder {
         slot: u64,
         fec_rate: f32,
         data_shred_batch: &[Shred],
+        version: u16,
     ) -> Vec<Shred> {
         assert!(!data_shred_batch.is_empty());
         if fec_rate != 0.0 {
@@ -526,6 +563,7 @@ impl Shredder {
                     num_data,
                     num_coding,
                     i,
+                    version,
                 );
                 let shred =
                     Shred::new_empty_from_header(header, DataShredHeader::default(), coding_header);
@@ -555,6 +593,7 @@ impl Shredder {
                         num_data,
                         num_coding,
                         i,
+                        version,
                     );
                     Shred {
                         common_header,
@@ -758,6 +797,7 @@ pub mod tests {
     use super::*;
     use bincode::serialized_size;
     use matches::assert_matches;
+    use solana_sdk::hash::hash;
     use solana_sdk::system_transaction;
     use std::collections::HashSet;
     use std::convert::TryInto;
@@ -825,7 +865,7 @@ pub mod tests {
 
         // Test that parent cannot be > current slot
         assert_matches!(
-            Shredder::new(slot, slot + 1, 1.00, keypair.clone(), 0),
+            Shredder::new(slot, slot + 1, 1.00, keypair.clone(), 0, 0),
             Err(ShredError::SlotTooLow {
                 slot: _,
                 parent_slot: _,
@@ -833,7 +873,7 @@ pub mod tests {
         );
         // Test that slot - parent cannot be > u16 MAX
         assert_matches!(
-            Shredder::new(slot, slot - 1 - 0xffff, 1.00, keypair.clone(), 0),
+            Shredder::new(slot, slot - 1 - 0xffff, 1.00, keypair.clone(), 0, 0),
             Err(ShredError::SlotTooLow {
                 slot: _,
                 parent_slot: _,
@@ -842,7 +882,7 @@ pub mod tests {
 
         let fec_rate = 0.25;
         let parent_slot = slot - 5;
-        let shredder = Shredder::new(slot, parent_slot, fec_rate, keypair.clone(), 0)
+        let shredder = Shredder::new(slot, parent_slot, fec_rate, keypair.clone(), 0, 0)
             .expect("Failed in creating shredder");
 
         let entries: Vec<_> = (0..5)
@@ -917,7 +957,7 @@ pub mod tests {
         let slot = 1;
 
         let parent_slot = 0;
-        let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), 0)
+        let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), 0, 0)
             .expect("Failed in creating shredder");
 
         let entries: Vec<_> = (0..5)
@@ -943,7 +983,7 @@ pub mod tests {
         let slot = 1;
 
         let parent_slot = 0;
-        let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), 5)
+        let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), 5, 0)
             .expect("Failed in creating shredder");
 
         let entries: Vec<_> = (0..5)
@@ -973,7 +1013,7 @@ pub mod tests {
         let slot = 1;
 
         let parent_slot = 0;
-        let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), u8::max_value())
+        let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), u8::max_value(), 0)
             .expect("Failed in creating shredder");
 
         let entries: Vec<_> = (0..5)
@@ -1010,11 +1050,11 @@ pub mod tests {
         let slot = 0x123456789abcdef0;
         // Test that FEC rate cannot be > 1.0
         assert_matches!(
-            Shredder::new(slot, slot - 5, 1.001, keypair.clone(), 0),
+            Shredder::new(slot, slot - 5, 1.001, keypair.clone(), 0, 0),
             Err(ShredError::InvalidFecRate(_))
         );
 
-        let shredder = Shredder::new(0x123456789abcdef0, slot - 5, 1.0, keypair.clone(), 0)
+        let shredder = Shredder::new(0x123456789abcdef0, slot - 5, 1.0, keypair.clone(), 0, 0)
             .expect("Failed in creating shredder");
 
         // Create enough entries to make > 1 shred
@@ -1056,7 +1096,7 @@ pub mod tests {
     fn test_recovery_and_reassembly() {
         let keypair = Arc::new(Keypair::new());
         let slot = 0x123456789abcdef0;
-        let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone(), 0)
+        let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone(), 0, 0)
             .expect("Failed in creating shredder");
 
         let keypair0 = Keypair::new();
@@ -1302,7 +1342,7 @@ pub mod tests {
     fn test_multi_fec_block_coding() {
         let keypair = Arc::new(Keypair::new());
         let slot = 0x123456789abcdef0;
-        let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone(), 0)
+        let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone(), 0, 0)
             .expect("Failed in creating shredder");
 
         let num_fec_sets = 100;
@@ -1385,4 +1425,54 @@ pub mod tests {
         let result = Shredder::deshred(&all_shreds[..]).unwrap();
         assert_eq!(serialized_entries[..], result[..serialized_entries.len()]);
     }
+
+    #[test]
+    fn test_shred_version() {
+        let keypair = Arc::new(Keypair::new());
+        let hash = hash(Hash::default().as_ref());
+        let version = Shred::version_from_hash(&hash);
+        assert_ne!(version, 0);
+        let shredder =
+            Shredder::new(0, 0, 1.0, keypair, 0, version).expect("Failed in creating shredder");
+
+        let entries: Vec<_> = (0..5)
+            .map(|_| {
+                let keypair0 = Keypair::new();
+                let keypair1 = Keypair::new();
+                let tx0 =
+                    system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default());
+                Entry::new(&Hash::default(), 1, vec![tx0])
+            })
+            .collect();
+
+        let (data_shreds, coding_shreds, _next_index) =
+            shredder.entries_to_shreds(&entries, true, 0);
+        assert!(!data_shreds
+            .iter()
+            .chain(coding_shreds.iter())
+            .any(|s| s.version() != version));
+    }
+
+    #[test]
+    fn test_version_from_hash() {
+        let hash = [
+            0xa5u8, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5,
+            0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0x5a, 0x5a,
+            0xa5, 0xa5, 0x5a, 0x5a,
+        ];
+        let version = Shred::version_from_hash(&Hash::new(&hash));
+        assert_eq!(version, 0);
+        let hash = [
+            0xa5u8, 0xa5, 0x5a, 0x5a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            0, 0, 0, 0, 0, 0, 0, 0,
+        ];
+        let version = Shred::version_from_hash(&Hash::new(&hash));
+        assert_eq!(version, 0xffff);
+        let hash = [
+            0xa5u8, 0xa5, 0x5a, 0x5a, 0xa5, 0xa5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        ];
+        let version = Shred::version_from_hash(&Hash::new(&hash));
+        assert_eq!(version, 0x5a5a);
+    }
 }

ledger/tests/blocktree.rs

@@ -20,7 +20,7 @@ fn test_multiple_threads_insert_shred() {
         let threads: Vec<_> = (0..num_threads)
             .map(|i| {
                 let entries = entry::create_ticks(1, Hash::default());
-                let shreds = blocktree::entries_to_test_shreds(entries, i + 1, 0, false);
+                let shreds = blocktree::entries_to_test_shreds(entries, i + 1, 0, false, 0);
                 let blocktree_ = blocktree.clone();
                 Builder::new()
                     .name("blocktree-writer".to_string())