gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Shredder no longer holds a keypair

Browse Source
This commit is contained in:
Michael Vines
2021-06-21 13:12:38 -07:00
parent 2435ea3ad8
commit 84b9de8c18
13 changed files with 178 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
core/benches/retransmit_stage.rs
View File

@ -83,11 +83,11 @@ fn bench_retransmitter(bencher: &mut Bencher) {
}) })
.collect(); .collect();
let keypair = Arc::new(Keypair::new()); let keypair = Keypair::new();
let slot = 0; let slot = 0;
let parent = 0; let parent = 0;
let shredder = Shredder::new(slot, parent, keypair, 0, 0).unwrap(); let shredder = Shredder::new(slot, parent, 0, 0).unwrap();
let mut data_shreds = shredder.entries_to_shreds(&entries, true, 0).0; let mut data_shreds = shredder.entries_to_shreds(&keypair, &entries, true, 0).0;
let num_packets = data_shreds.len(); let num_packets = data_shreds.len();

22
core/benches/shredder.rs
View File

@ -14,7 +14,6 @@ use solana_ledger::shred::{
use solana_perf::test_tx; use solana_perf::test_tx;
use solana_sdk::hash::Hash; use solana_sdk::hash::Hash;
use solana_sdk::signature::Keypair; use solana_sdk::signature::Keypair;
use std::sync::Arc;
use test::Bencher; use test::Bencher;
fn make_test_entry(txs_per_entry: u64) -> Entry { fn make_test_entry(txs_per_entry: u64) -> Entry {
@ -39,9 +38,10 @@ fn make_shreds(num_shreds: usize) -> Vec<Shred> {
Some(shred_size), Some(shred_size),
); );
let entries = make_large_unchained_entries(txs_per_entry, num_entries); let entries = make_large_unchained_entries(txs_per_entry, num_entries);
let shredder = Shredder::new(1, 0, Arc::new(Keypair::new()), 0, 0).unwrap(); let shredder = Shredder::new(1, 0, 0, 0).unwrap();
let data_shreds = shredder let data_shreds = shredder
.entries_to_data_shreds( .entries_to_data_shreds(
&Keypair::new(),
&entries, &entries,
true, // is_last_in_slot true, // is_last_in_slot
0, // next_shred_index 0, // next_shred_index
@ -67,21 +67,21 @@ fn make_concatenated_shreds(num_shreds: usize) -> Vec<u8> {
#[bench] #[bench]
fn bench_shredder_ticks(bencher: &mut Bencher) { fn bench_shredder_ticks(bencher: &mut Bencher) {
let kp = Arc::new(Keypair::new()); let kp = Keypair::new();
let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD; let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD;
let num_shreds = ((1000 * 1000) + (shred_size - 1)) / shred_size; let num_shreds = ((1000 * 1000) + (shred_size - 1)) / shred_size;
// ~1Mb // ~1Mb
let num_ticks = max_ticks_per_n_shreds(1, Some(SIZE_OF_DATA_SHRED_PAYLOAD)) * num_shreds as u64; let num_ticks = max_ticks_per_n_shreds(1, Some(SIZE_OF_DATA_SHRED_PAYLOAD)) * num_shreds as u64;
let entries = create_ticks(num_ticks, 0, Hash::default()); let entries = create_ticks(num_ticks, 0, Hash::default());
bencher.iter(|| { bencher.iter(|| {
let shredder = Shredder::new(1, 0, kp.clone(), 0, 0).unwrap(); let shredder = Shredder::new(1, 0, 0, 0).unwrap();
shredder.entries_to_shreds(&entries, true, 0); shredder.entries_to_shreds(&kp, &entries, true, 0);
}) })
} }
#[bench] #[bench]
fn bench_shredder_large_entries(bencher: &mut Bencher) { fn bench_shredder_large_entries(bencher: &mut Bencher) {
let kp = Arc::new(Keypair::new()); let kp = Keypair::new();
let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD; let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD;
let num_shreds = ((1000 * 1000) + (shred_size - 1)) / shred_size; let num_shreds = ((1000 * 1000) + (shred_size - 1)) / shred_size;
let txs_per_entry = 128; let txs_per_entry = 128;
@ -93,21 +93,21 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
let entries = make_large_unchained_entries(txs_per_entry, num_entries); let entries = make_large_unchained_entries(txs_per_entry, num_entries);
// 1Mb // 1Mb
bencher.iter(|| { bencher.iter(|| {
let shredder = Shredder::new(1, 0, kp.clone(), 0, 0).unwrap(); let shredder = Shredder::new(1, 0, 0, 0).unwrap();
shredder.entries_to_shreds(&entries, true, 0); shredder.entries_to_shreds(&kp, &entries, true, 0);
}) })
} }
#[bench] #[bench]
fn bench_deshredder(bencher: &mut Bencher) { fn bench_deshredder(bencher: &mut Bencher) {
let kp = Arc::new(Keypair::new()); let kp = Keypair::new();
let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD; let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD;
// ~10Mb // ~10Mb
let num_shreds = ((10000 * 1000) + (shred_size - 1)) / shred_size; let num_shreds = ((10000 * 1000) + (shred_size - 1)) / shred_size;
let num_ticks = max_ticks_per_n_shreds(1, Some(shred_size)) * num_shreds as u64; let num_ticks = max_ticks_per_n_shreds(1, Some(shred_size)) * num_shreds as u64;
let entries = create_ticks(num_ticks, 0, Hash::default()); let entries = create_ticks(num_ticks, 0, Hash::default());
let shredder = Shredder::new(1, 0, kp, 0, 0).unwrap(); let shredder = Shredder::new(1, 0, 0, 0).unwrap();
let data_shreds = shredder.entries_to_shreds(&entries, true, 0).0; let data_shreds = shredder.entries_to_shreds(&kp, &entries, true, 0).0;
bencher.iter(|| { bencher.iter(|| {
let raw = &mut Shredder::deshred(&data_shreds).unwrap(); let raw = &mut Shredder::deshred(&data_shreds).unwrap();
assert_ne!(raw.len(), 0); assert_ne!(raw.len(), 0);

27
core/src/broadcast_stage.rs
View File

@ -23,7 +23,7 @@ use solana_metrics::{inc_new_counter_error, inc_new_counter_info};
use solana_poh::poh_recorder::WorkingBankEntry; use solana_poh::poh_recorder::WorkingBankEntry;
use solana_runtime::bank::Bank; use solana_runtime::bank::Bank;
use solana_sdk::timing::timestamp; use solana_sdk::timing::timestamp;
use solana_sdk::{clock::Slot, pubkey::Pubkey}; use solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Keypair};
use solana_streamer::sendmmsg::send_mmsg; use solana_streamer::sendmmsg::send_mmsg;
use std::sync::atomic::AtomicU64; use std::sync::atomic::AtomicU64;
use std::{ use std::{
@ -81,7 +81,6 @@ impl BroadcastStageType {
blockstore: &Arc<Blockstore>, blockstore: &Arc<Blockstore>,
shred_version: u16, shred_version: u16,
) -> BroadcastStage { ) -> BroadcastStage {
let keypair = cluster_info.keypair.clone();
match self { match self {
BroadcastStageType::Standard => BroadcastStage::new( BroadcastStageType::Standard => BroadcastStage::new(
sock, sock,
@ -90,7 +89,7 @@ impl BroadcastStageType {
retransmit_slots_receiver, retransmit_slots_receiver,
exit_sender, exit_sender,
blockstore, blockstore,
StandardBroadcastRun::new(keypair, shred_version), StandardBroadcastRun::new(shred_version),
), ),
BroadcastStageType::FailEntryVerification => BroadcastStage::new( BroadcastStageType::FailEntryVerification => BroadcastStage::new(
@ -100,7 +99,7 @@ impl BroadcastStageType {
retransmit_slots_receiver, retransmit_slots_receiver,
exit_sender, exit_sender,
blockstore, blockstore,
FailEntryVerificationBroadcastRun::new(keypair, shred_version), FailEntryVerificationBroadcastRun::new(shred_version),
), ),
BroadcastStageType::BroadcastFakeShreds => BroadcastStage::new( BroadcastStageType::BroadcastFakeShreds => BroadcastStage::new(
@ -110,7 +109,7 @@ impl BroadcastStageType {
retransmit_slots_receiver, retransmit_slots_receiver,
exit_sender, exit_sender,
blockstore, blockstore,
BroadcastFakeShredsRun::new(keypair, 0, shred_version), BroadcastFakeShredsRun::new(0, shred_version),
), ),
BroadcastStageType::BroadcastDuplicates(config) => BroadcastStage::new( BroadcastStageType::BroadcastDuplicates(config) => BroadcastStage::new(
@ -120,7 +119,7 @@ impl BroadcastStageType {
retransmit_slots_receiver, retransmit_slots_receiver,
exit_sender, exit_sender,
blockstore, blockstore,
BroadcastDuplicatesRun::new(keypair, shred_version, config.clone()), BroadcastDuplicatesRun::new(shred_version, config.clone()),
), ),
} }
} }
@ -130,6 +129,7 @@ pub type TransmitShreds = (Option<Arc<HashMap<Pubkey, u64>>>, Arc<Vec<Shred>>);
trait BroadcastRun { trait BroadcastRun {
fn run( fn run(
&mut self, &mut self,
keypair: &Keypair,
blockstore: &Arc<Blockstore>, blockstore: &Arc<Blockstore>,
receiver: &Receiver<WorkingBankEntry>, receiver: &Receiver<WorkingBankEntry>,
socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>, socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
@ -173,6 +173,7 @@ pub struct BroadcastStage {
impl BroadcastStage { impl BroadcastStage {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
fn run( fn run(
keypair: &Keypair,
blockstore: &Arc<Blockstore>, blockstore: &Arc<Blockstore>,
receiver: &Receiver<WorkingBankEntry>, receiver: &Receiver<WorkingBankEntry>,
socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>, socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
@ -180,8 +181,13 @@ impl BroadcastStage {
mut broadcast_stage_run: impl BroadcastRun, mut broadcast_stage_run: impl BroadcastRun,
) -> BroadcastStageReturnType { ) -> BroadcastStageReturnType {
loop { loop {
let res = let res = broadcast_stage_run.run(
broadcast_stage_run.run(blockstore, receiver, socket_sender, blockstore_sender); keypair,
blockstore,
receiver,
socket_sender,
blockstore_sender,
);
let res = Self::handle_error(res, "run"); let res = Self::handle_error(res, "run");
if let Some(res) = res { if let Some(res) = res {
return res; return res;
@ -242,11 +248,13 @@ impl BroadcastStage {
let bs_run = broadcast_stage_run.clone(); let bs_run = broadcast_stage_run.clone();
let socket_sender_ = socket_sender.clone(); let socket_sender_ = socket_sender.clone();
let keypair = cluster_info.keypair.clone();
let thread_hdl = Builder::new() let thread_hdl = Builder::new()
.name("solana-broadcaster".to_string()) .name("solana-broadcaster".to_string())
.spawn(move || { .spawn(move || {
let _finalizer = Finalizer::new(exit); let _finalizer = Finalizer::new(exit);
Self::run( Self::run(
&keypair,
&btree, &btree,
&receiver, &receiver,
&socket_sender_, &socket_sender_,
@ -635,7 +643,6 @@ pub mod test {
let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000); let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
let bank = Arc::new(Bank::new(&genesis_config)); let bank = Arc::new(Bank::new(&genesis_config));
let leader_keypair = cluster_info.keypair.clone();
// Start up the broadcast stage // Start up the broadcast stage
let broadcast_service = BroadcastStage::new( let broadcast_service = BroadcastStage::new(
leader_info.sockets.broadcast, leader_info.sockets.broadcast,
@ -644,7 +651,7 @@ pub mod test {
retransmit_slots_receiver, retransmit_slots_receiver,
&exit_sender, &exit_sender,
&blockstore, &blockstore,
StandardBroadcastRun::new(leader_keypair, 0), StandardBroadcastRun::new(0),
); );
MockBroadcastStage { MockBroadcastStage {

16
core/src/broadcast_stage/broadcast_duplicates_run.rs
View File

@ -28,15 +28,10 @@ pub(super) struct BroadcastDuplicatesRun {
last_broadcast_slot: Slot, last_broadcast_slot: Slot,
next_shred_index: u32, next_shred_index: u32,
shred_version: u16, shred_version: u16,
keypair: Arc<Keypair>,
} }
impl BroadcastDuplicatesRun { impl BroadcastDuplicatesRun {
pub(super) fn new( pub(super) fn new(shred_version: u16, config: BroadcastDuplicatesConfig) -> Self {
keypair: Arc<Keypair>,
shred_version: u16,
config: BroadcastDuplicatesConfig,
) -> Self {
let mut delayed_queue = DelayedQueue::new(); let mut delayed_queue = DelayedQueue::new();
delayed_queue.resize(config.duplicate_send_delay, (None, None)); delayed_queue.resize(config.duplicate_send_delay, (None, None));
Self { Self {
@ -48,7 +43,6 @@ impl BroadcastDuplicatesRun {
last_broadcast_slot: 0, last_broadcast_slot: 0,
last_duplicate_entry_hash: Hash::default(), last_duplicate_entry_hash: Hash::default(),
shred_version, shred_version,
keypair,
} }
} }
@ -139,6 +133,7 @@ pub const MINIMUM_DUPLICATE_SLOT: Slot = 20;
impl BroadcastRun for BroadcastDuplicatesRun { impl BroadcastRun for BroadcastDuplicatesRun {
fn run( fn run(
&mut self, &mut self,
keypair: &Keypair,
blockstore: &Arc<Blockstore>, blockstore: &Arc<Blockstore>,
receiver: &Receiver<WorkingBankEntry>, receiver: &Receiver<WorkingBankEntry>,
socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>, socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
@ -166,13 +161,13 @@ impl BroadcastRun for BroadcastDuplicatesRun {
let shredder = Shredder::new( let shredder = Shredder::new(
bank.slot(), bank.slot(),
bank.parent().unwrap().slot(), bank.parent().unwrap().slot(),
self.keypair.clone(),
(bank.tick_height() % bank.ticks_per_slot()) as u8, (bank.tick_height() % bank.ticks_per_slot()) as u8,
self.shred_version, self.shred_version,
) )
.expect("Expected to create a new shredder"); .expect("Expected to create a new shredder");
let (data_shreds, coding_shreds, last_shred_index) = shredder.entries_to_shreds( let (data_shreds, coding_shreds, last_shred_index) = shredder.entries_to_shreds(
keypair,
&receive_results.entries, &receive_results.entries,
last_tick_height == bank.max_tick_height(), last_tick_height == bank.max_tick_height(),
self.next_shred_index, self.next_shred_index,
@ -182,6 +177,7 @@ impl BroadcastRun for BroadcastDuplicatesRun {
self.queue_or_create_duplicate_entries(&bank, &receive_results); self.queue_or_create_duplicate_entries(&bank, &receive_results);
let (duplicate_data_shreds, duplicate_coding_shreds, _) = if !duplicate_entries.is_empty() { let (duplicate_data_shreds, duplicate_coding_shreds, _) = if !duplicate_entries.is_empty() {
shredder.entries_to_shreds( shredder.entries_to_shreds(
keypair,
&duplicate_entries, &duplicate_entries,
last_tick_height == bank.max_tick_height(), last_tick_height == bank.max_tick_height(),
next_duplicate_shred_index, next_duplicate_shred_index,
@ -208,7 +204,7 @@ impl BroadcastRun for BroadcastDuplicatesRun {
.epoch_staked_nodes(bank_epoch) .epoch_staked_nodes(bank_epoch)
.unwrap() .unwrap()
.into_iter() .into_iter()
.filter(|(pubkey, _)| *pubkey != self.keypair.pubkey()) .filter(|(pubkey, _)| *pubkey != keypair.pubkey())
.collect(); .collect();
stakes.sort_by(|(l_key, l_stake), (r_key, r_stake)| { stakes.sort_by(|(l_key, l_stake), (r_key, r_stake)| {
if r_stake == l_stake { if r_stake == l_stake {
@ -234,7 +230,7 @@ impl BroadcastRun for BroadcastDuplicatesRun {
if bank.slot() > MINIMUM_DUPLICATE_SLOT && last_tick_height == bank.max_tick_height() { if bank.slot() > MINIMUM_DUPLICATE_SLOT && last_tick_height == bank.max_tick_height() {
warn!( warn!(
"{} sent duplicate slot {} to nodes: {:?}", "{} sent duplicate slot {} to nodes: {:?}",
self.keypair.pubkey(), keypair.pubkey(),
bank.slot(), bank.slot(),
&duplicate_recipients, &duplicate_recipients,
); );

8
core/src/broadcast_stage/broadcast_fake_shreds_run.rs
View File

@ -9,16 +9,14 @@ pub(super) struct BroadcastFakeShredsRun {
last_blockhash: Hash, last_blockhash: Hash,
partition: usize, partition: usize,
shred_version: u16, shred_version: u16,
keypair: Arc<Keypair>,
} }
impl BroadcastFakeShredsRun { impl BroadcastFakeShredsRun {
pub(super) fn new(keypair: Arc<Keypair>, partition: usize, shred_version: u16) -> Self { pub(super) fn new(partition: usize, shred_version: u16) -> Self {
Self { Self {
last_blockhash: Hash::default(), last_blockhash: Hash::default(),
partition, partition,
shred_version, shred_version,
keypair,
} }
} }
} }
@ -26,6 +24,7 @@ impl BroadcastFakeShredsRun {
impl BroadcastRun for BroadcastFakeShredsRun { impl BroadcastRun for BroadcastFakeShredsRun {
fn run( fn run(
&mut self, &mut self,
keypair: &Keypair,
blockstore: &Arc<Blockstore>, blockstore: &Arc<Blockstore>,
receiver: &Receiver<WorkingBankEntry>, receiver: &Receiver<WorkingBankEntry>,
socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>, socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
@ -47,13 +46,13 @@ impl BroadcastRun for BroadcastFakeShredsRun {
let shredder = Shredder::new( let shredder = Shredder::new(
bank.slot(), bank.slot(),
bank.parent().unwrap().slot(), bank.parent().unwrap().slot(),
self.keypair.clone(),
(bank.tick_height() % bank.ticks_per_slot()) as u8, (bank.tick_height() % bank.ticks_per_slot()) as u8,
self.shred_version, self.shred_version,
) )
.expect("Expected to create a new shredder"); .expect("Expected to create a new shredder");
let (data_shreds, coding_shreds, _) = shredder.entries_to_shreds( let (data_shreds, coding_shreds, _) = shredder.entries_to_shreds(
keypair,
&receive_results.entries, &receive_results.entries,
last_tick_height == bank.max_tick_height(), last_tick_height == bank.max_tick_height(),
next_shred_index, next_shred_index,
@ -70,6 +69,7 @@ impl BroadcastRun for BroadcastFakeShredsRun {
.collect(); .collect();
let (fake_data_shreds, fake_coding_shreds, _) = shredder.entries_to_shreds( let (fake_data_shreds, fake_coding_shreds, _) = shredder.entries_to_shreds(
keypair,
&fake_entries, &fake_entries,
last_tick_height == bank.max_tick_height(), last_tick_height == bank.max_tick_height(),
next_shred_index, next_shred_index,

11
core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs
View File

@ -10,17 +10,15 @@ pub const SLOT_TO_RESOLVE: u64 = 32;
#[derive(Clone)] #[derive(Clone)]
pub(super) struct FailEntryVerificationBroadcastRun { pub(super) struct FailEntryVerificationBroadcastRun {
shred_version: u16, shred_version: u16,
keypair: Arc<Keypair>,
good_shreds: Vec<Shred>, good_shreds: Vec<Shred>,
current_slot: Slot, current_slot: Slot,
next_shred_index: u32, next_shred_index: u32,
} }
impl FailEntryVerificationBroadcastRun { impl FailEntryVerificationBroadcastRun {
pub(super) fn new(keypair: Arc<Keypair>, shred_version: u16) -> Self { pub(super) fn new(shred_version: u16) -> Self {
Self { Self {
shred_version, shred_version,
keypair,
good_shreds: vec![], good_shreds: vec![],
current_slot: 0, current_slot: 0,
next_shred_index: 0, next_shred_index: 0,
@ -31,6 +29,7 @@ impl FailEntryVerificationBroadcastRun {
impl BroadcastRun for FailEntryVerificationBroadcastRun { impl BroadcastRun for FailEntryVerificationBroadcastRun {
fn run( fn run(
&mut self, &mut self,
keypair: &Keypair,
blockstore: &Arc<Blockstore>, blockstore: &Arc<Blockstore>,
receiver: &Receiver<WorkingBankEntry>, receiver: &Receiver<WorkingBankEntry>,
socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>, socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
@ -71,13 +70,13 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
let shredder = Shredder::new( let shredder = Shredder::new(
bank.slot(), bank.slot(),
bank.parent().unwrap().slot(), bank.parent().unwrap().slot(),
self.keypair.clone(),
(bank.tick_height() % bank.ticks_per_slot()) as u8, (bank.tick_height() % bank.ticks_per_slot()) as u8,
self.shred_version, self.shred_version,
) )
.expect("Expected to create a new shredder"); .expect("Expected to create a new shredder");
let (data_shreds, _, _) = shredder.entries_to_shreds( let (data_shreds, _, _) = shredder.entries_to_shreds(
keypair,
&receive_results.entries, &receive_results.entries,
last_tick_height == bank.max_tick_height() && last_entries.is_none(), last_tick_height == bank.max_tick_height() && last_entries.is_none(),
self.next_shred_index, self.next_shred_index,
@ -86,12 +85,12 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
self.next_shred_index += data_shreds.len() as u32; self.next_shred_index += data_shreds.len() as u32;
let last_shreds = last_entries.map(|(good_last_entry, bad_last_entry)| { let last_shreds = last_entries.map(|(good_last_entry, bad_last_entry)| {
let (good_last_data_shred, _, _) = let (good_last_data_shred, _, _) =
shredder.entries_to_shreds(&[good_last_entry], true, self.next_shred_index); shredder.entries_to_shreds(keypair, &[good_last_entry], true, self.next_shred_index);
let (bad_last_data_shred, _, _) = let (bad_last_data_shred, _, _) =
// Don't mark the last shred as last so that validators won't know that // Don't mark the last shred as last so that validators won't know that
// they've gotten all the shreds, and will continue trying to repair // they've gotten all the shreds, and will continue trying to repair
shredder.entries_to_shreds(&[bad_last_entry], false, self.next_shred_index); shredder.entries_to_shreds(keypair, &[bad_last_entry], false, self.next_shred_index);
self.next_shred_index += 1; self.next_shred_index += 1;
(good_last_data_shred, bad_last_data_shred) (good_last_data_shred, bad_last_data_shred)

76
core/src/broadcast_stage/standard_broadcast_run.rs
View File

@ -13,7 +13,7 @@ use solana_ledger::{
}, },
}; };
use solana_sdk::{pubkey::Pubkey, signature::Keypair, timing::duration_as_us}; use solana_sdk::{pubkey::Pubkey, signature::Keypair, timing::duration_as_us};
use std::{collections::HashMap, ops::Deref, sync::RwLock, time::Duration}; use std::{collections::HashMap, sync::RwLock, time::Duration};
#[derive(Clone)] #[derive(Clone)]
pub struct StandardBroadcastRun { pub struct StandardBroadcastRun {
@ -23,7 +23,6 @@ pub struct StandardBroadcastRun {
unfinished_slot: Option<UnfinishedSlotInfo>, unfinished_slot: Option<UnfinishedSlotInfo>,
current_slot_and_parent: Option<(u64, u64)>, current_slot_and_parent: Option<(u64, u64)>,
slot_broadcast_start: Option<Instant>, slot_broadcast_start: Option<Instant>,
keypair: Arc<Keypair>,
shred_version: u16, shred_version: u16,
last_datapoint_submit: Arc<AtomicU64>, last_datapoint_submit: Arc<AtomicU64>,
num_batches: usize, num_batches: usize,
@ -38,7 +37,7 @@ struct BroadcastPeerCache {
} }
impl StandardBroadcastRun { impl StandardBroadcastRun {
pub(super) fn new(keypair: Arc<Keypair>, shred_version: u16) -> Self { pub(super) fn new(shred_version: u16) -> Self {
Self { Self {
process_shreds_stats: ProcessShredsStats::default(), process_shreds_stats: ProcessShredsStats::default(),
transmit_shreds_stats: Arc::default(), transmit_shreds_stats: Arc::default(),
@ -46,7 +45,6 @@ impl StandardBroadcastRun {
unfinished_slot: None, unfinished_slot: None,
current_slot_and_parent: None, current_slot_and_parent: None,
slot_broadcast_start: None, slot_broadcast_start: None,
keypair,
shred_version, shred_version,
last_datapoint_submit: Arc::default(), last_datapoint_submit: Arc::default(),
num_batches: 0, num_batches: 0,
@ -60,6 +58,7 @@ impl StandardBroadcastRun {
// shreds buffered. // shreds buffered.
fn finish_prev_slot( fn finish_prev_slot(
&mut self, &mut self,
keypair: &Keypair,
max_ticks_in_slot: u8, max_ticks_in_slot: u8,
stats: &mut ProcessShredsStats, stats: &mut ProcessShredsStats,
) -> Vec<Shred> { ) -> Vec<Shred> {
@ -83,10 +82,10 @@ impl StandardBroadcastRun {
self.shred_version, self.shred_version,
fec_set_index.unwrap(), fec_set_index.unwrap(),
); );
Shredder::sign_shred(self.keypair.deref(), &mut shred); Shredder::sign_shred(keypair, &mut shred);
state.data_shreds_buffer.push(shred.clone()); state.data_shreds_buffer.push(shred.clone());
let mut shreds = make_coding_shreds( let mut shreds = make_coding_shreds(
self.keypair.deref(), keypair,
&mut self.unfinished_slot, &mut self.unfinished_slot,
true, // is_last_in_slot true, // is_last_in_slot
stats, stats,
@ -101,6 +100,7 @@ impl StandardBroadcastRun {
fn entries_to_data_shreds( fn entries_to_data_shreds(
&mut self, &mut self,
keypair: &Keypair,
entries: &[Entry], entries: &[Entry],
blockstore: &Blockstore, blockstore: &Blockstore,
reference_tick: u8, reference_tick: u8,
@ -118,15 +118,11 @@ impl StandardBroadcastRun {
None => (0, 0), None => (0, 0),
}, },
}; };
let (data_shreds, next_shred_index) = Shredder::new( let (data_shreds, next_shred_index) =
slot, Shredder::new(slot, parent_slot, reference_tick, self.shred_version)
parent_slot,
self.keypair.clone(),
reference_tick,
self.shred_version,
)
.unwrap() .unwrap()
.entries_to_data_shreds( .entries_to_data_shreds(
keypair,
entries, entries,
is_slot_end, is_slot_end,
next_shred_index, next_shred_index,
@ -154,6 +150,7 @@ impl StandardBroadcastRun {
#[cfg(test)] #[cfg(test)]
fn test_process_receive_results( fn test_process_receive_results(
&mut self, &mut self,
keypair: &Keypair,
cluster_info: &ClusterInfo, cluster_info: &ClusterInfo,
sock: &UdpSocket, sock: &UdpSocket,
blockstore: &Arc<Blockstore>, blockstore: &Arc<Blockstore>,
@ -161,7 +158,7 @@ impl StandardBroadcastRun {
) -> Result<()> { ) -> Result<()> {
let (bsend, brecv) = channel(); let (bsend, brecv) = channel();
let (ssend, srecv) = channel(); let (ssend, srecv) = channel();
self.process_receive_results(blockstore, &ssend, &bsend, receive_results)?; self.process_receive_results(keypair, blockstore, &ssend, &bsend, receive_results)?;
let srecv = Arc::new(Mutex::new(srecv)); let srecv = Arc::new(Mutex::new(srecv));
let brecv = Arc::new(Mutex::new(brecv)); let brecv = Arc::new(Mutex::new(brecv));
//data //data
@ -175,6 +172,7 @@ impl StandardBroadcastRun {
fn process_receive_results( fn process_receive_results(
&mut self, &mut self,
keypair: &Keypair,
blockstore: &Arc<Blockstore>, blockstore: &Arc<Blockstore>,
socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>, socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>, blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
@ -205,12 +203,13 @@ impl StandardBroadcastRun {
// 1) Check if slot was interrupted // 1) Check if slot was interrupted
let prev_slot_shreds = let prev_slot_shreds =
self.finish_prev_slot(bank.ticks_per_slot() as u8, &mut process_stats); self.finish_prev_slot(keypair, bank.ticks_per_slot() as u8, &mut process_stats);
// 2) Convert entries to shreds and coding shreds // 2) Convert entries to shreds and coding shreds
let is_last_in_slot = last_tick_height == bank.max_tick_height(); let is_last_in_slot = last_tick_height == bank.max_tick_height();
let reference_tick = bank.tick_height() % bank.ticks_per_slot(); let reference_tick = bank.tick_height() % bank.ticks_per_slot();
let data_shreds = self.entries_to_data_shreds( let data_shreds = self.entries_to_data_shreds(
keypair,
&receive_results.entries, &receive_results.entries,
blockstore, blockstore,
reference_tick as u8, reference_tick as u8,
@ -273,7 +272,7 @@ impl StandardBroadcastRun {
// Create and send coding shreds // Create and send coding shreds
let coding_shreds = make_coding_shreds( let coding_shreds = make_coding_shreds(
self.keypair.deref(), keypair,
&mut self.unfinished_slot, &mut self.unfinished_slot,
is_last_in_slot, is_last_in_slot,
&mut process_stats, &mut process_stats,
@ -456,6 +455,7 @@ fn make_coding_shreds(
impl BroadcastRun for StandardBroadcastRun { impl BroadcastRun for StandardBroadcastRun {
fn run( fn run(
&mut self, &mut self,
keypair: &Keypair,
blockstore: &Arc<Blockstore>, blockstore: &Arc<Blockstore>,
receiver: &Receiver<WorkingBankEntry>, receiver: &Receiver<WorkingBankEntry>,
socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>, socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
@ -465,6 +465,7 @@ impl BroadcastRun for StandardBroadcastRun {
// TODO: Confirm that last chunk of coding shreds // TODO: Confirm that last chunk of coding shreds
// will not be lost or delayed for too long. // will not be lost or delayed for too long.
self.process_receive_results( self.process_receive_results(
keypair,
blockstore, blockstore,
socket_sender, socket_sender,
blockstore_sender, blockstore_sender,
@ -505,6 +506,7 @@ mod test {
genesis_config::GenesisConfig, genesis_config::GenesisConfig,
signature::{Keypair, Signer}, signature::{Keypair, Signer},
}; };
use std::ops::Deref;
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
@ -544,7 +546,7 @@ mod test {
#[test] #[test]
fn test_interrupted_slot_last_shred() { fn test_interrupted_slot_last_shred() {
let keypair = Arc::new(Keypair::new()); let keypair = Arc::new(Keypair::new());
let mut run = StandardBroadcastRun::new(keypair.clone(), 0); let mut run = StandardBroadcastRun::new(0);
// Set up the slot to be interrupted // Set up the slot to be interrupted
let next_shred_index = 10; let next_shred_index = 10;
@ -563,7 +565,7 @@ mod test {
run.current_slot_and_parent = Some((4, 2)); run.current_slot_and_parent = Some((4, 2));
// Slot 2 interrupted slot 1 // Slot 2 interrupted slot 1
let shreds = run.finish_prev_slot(0, &mut ProcessShredsStats::default()); let shreds = run.finish_prev_slot(&keypair, 0, &mut ProcessShredsStats::default());
let shred = shreds let shred = shreds
.get(0) .get(0)
.expect("Expected a shred that signals an interrupt"); .expect("Expected a shred that signals an interrupt");
@ -593,9 +595,15 @@ mod test {
}; };
// Step 1: Make an incomplete transmission for slot 0 // Step 1: Make an incomplete transmission for slot 0
let mut standard_broadcast_run = StandardBroadcastRun::new(leader_keypair.clone(), 0); let mut standard_broadcast_run = StandardBroadcastRun::new(0);
standard_broadcast_run standard_broadcast_run
.test_process_receive_results(&cluster_info, &socket, &blockstore, receive_results) .test_process_receive_results(
&leader_keypair,
&cluster_info,
&socket,
&blockstore,
receive_results,
)
.unwrap(); .unwrap();
let unfinished_slot = standard_broadcast_run.unfinished_slot.as_ref().unwrap(); let unfinished_slot = standard_broadcast_run.unfinished_slot.as_ref().unwrap();
assert_eq!(unfinished_slot.next_shred_index as u64, num_shreds_per_slot); assert_eq!(unfinished_slot.next_shred_index as u64, num_shreds_per_slot);
@ -653,7 +661,13 @@ mod test {
last_tick_height: (ticks1.len() - 1) as u64, last_tick_height: (ticks1.len() - 1) as u64,
}; };
standard_broadcast_run standard_broadcast_run
.test_process_receive_results(&cluster_info, &socket, &blockstore, receive_results) .test_process_receive_results(
&leader_keypair,
&cluster_info,
&socket,
&blockstore,
receive_results,
)
.unwrap(); .unwrap();
let unfinished_slot = standard_broadcast_run.unfinished_slot.as_ref().unwrap(); let unfinished_slot = standard_broadcast_run.unfinished_slot.as_ref().unwrap();
@ -699,7 +713,7 @@ mod test {
let (bsend, brecv) = channel(); let (bsend, brecv) = channel();
let (ssend, _srecv) = channel(); let (ssend, _srecv) = channel();
let mut last_tick_height = 0; let mut last_tick_height = 0;
let mut standard_broadcast_run = StandardBroadcastRun::new(leader_keypair, 0); let mut standard_broadcast_run = StandardBroadcastRun::new(0);
let mut process_ticks = |num_ticks| { let mut process_ticks = |num_ticks| {
let ticks = create_ticks(num_ticks, 0, genesis_config.hash()); let ticks = create_ticks(num_ticks, 0, genesis_config.hash());
last_tick_height += (ticks.len() - 1) as u64; last_tick_height += (ticks.len() - 1) as u64;
@ -710,7 +724,13 @@ mod test {
last_tick_height, last_tick_height,
}; };
standard_broadcast_run standard_broadcast_run
.process_receive_results(&blockstore, &ssend, &bsend, receive_results) .process_receive_results(
&leader_keypair,
&blockstore,
&ssend,
&bsend,
receive_results,
)
.unwrap(); .unwrap();
}; };
for i in 0..3 { for i in 0..3 {
@ -751,9 +771,15 @@ mod test {
last_tick_height: ticks.len() as u64, last_tick_height: ticks.len() as u64,
}; };
let mut standard_broadcast_run = StandardBroadcastRun::new(leader_keypair, 0); let mut standard_broadcast_run = StandardBroadcastRun::new(0);
standard_broadcast_run standard_broadcast_run
.test_process_receive_results(&cluster_info, &socket, &blockstore, receive_results) .test_process_receive_results(
&leader_keypair,
&cluster_info,
&socket,
&blockstore,
receive_results,
)
.unwrap(); .unwrap();
assert!(standard_broadcast_run.unfinished_slot.is_none()) assert!(standard_broadcast_run.unfinished_slot.is_none())
} }

8
core/src/window_service.rs
View File

@ -627,10 +627,10 @@ mod test {
entries: &[Entry], entries: &[Entry],
slot: Slot, slot: Slot,
parent: Slot, parent: Slot,
keypair: &Arc<Keypair>, keypair: &Keypair,
) -> Vec<Shred> { ) -> Vec<Shred> {
let shredder = Shredder::new(slot, parent, keypair.clone(), 0, 0).unwrap(); let shredder = Shredder::new(slot, parent, 0, 0).unwrap();
shredder.entries_to_shreds(entries, true, 0).0 shredder.entries_to_shreds(keypair, entries, true, 0).0
} }
#[test] #[test]
@ -639,7 +639,7 @@ mod test {
let blockstore = Arc::new(Blockstore::open(&blockstore_path).unwrap()); let blockstore = Arc::new(Blockstore::open(&blockstore_path).unwrap());
let num_entries = 10; let num_entries = 10;
let original_entries = create_ticks(num_entries, 0, Hash::default()); let original_entries = create_ticks(num_entries, 0, Hash::default());
let mut shreds = local_entries_to_shred(&original_entries, 0, 0, &Arc::new(Keypair::new())); let mut shreds = local_entries_to_shred(&original_entries, 0, 0, &Keypair::new());
shreds.reverse(); shreds.reverse();
blockstore blockstore
.insert_shreds(shreds, None, false) .insert_shreds(shreds, None, false)

7
gossip/src/cluster_info.rs
View File

@ -3408,11 +3408,10 @@ mod tests {
let leader = Arc::new(Keypair::new()); let leader = Arc::new(Keypair::new());
let keypair = Keypair::new(); let keypair = Keypair::new();
let (slot, parent_slot, reference_tick, version) = (53084024, 53084023, 0, 0); let (slot, parent_slot, reference_tick, version) = (53084024, 53084023, 0, 0);
let shredder = let shredder = Shredder::new(slot, parent_slot, reference_tick, version).unwrap();
Shredder::new(slot, parent_slot, leader.clone(), reference_tick, version).unwrap();
let next_shred_index = rng.gen(); let next_shred_index = rng.gen();
let shred = new_rand_shred(&mut rng, next_shred_index, &shredder); let shred = new_rand_shred(&mut rng, next_shred_index, &shredder, &leader);
let other_payload = new_rand_shred(&mut rng, next_shred_index, &shredder).payload; let other_payload = new_rand_shred(&mut rng, next_shred_index, &shredder, &leader).payload;
let leader_schedule = |s| { let leader_schedule = |s| {
if s == slot { if s == slot {
Some(leader.pubkey()) Some(leader.pubkey())

9
gossip/src/duplicate_shred.rs
View File

@ -317,6 +317,7 @@ pub(crate) mod tests {
rng: &mut R, rng: &mut R,
next_shred_index: u32, next_shred_index: u32,
shredder: &Shredder, shredder: &Shredder,
keypair: &Keypair,
) -> Shred { ) -> Shred {
let entries: Vec<_> = std::iter::repeat_with(|| { let entries: Vec<_> = std::iter::repeat_with(|| {
let tx = system_transaction::transfer( let tx = system_transaction::transfer(
@ -334,6 +335,7 @@ pub(crate) mod tests {
.take(5) .take(5)
.collect(); .collect();
let (mut data_shreds, _coding_shreds, _last_shred_index) = shredder.entries_to_shreds( let (mut data_shreds, _coding_shreds, _last_shred_index) = shredder.entries_to_shreds(
keypair,
&entries, &entries,
true, // is_last_in_slot true, // is_last_in_slot
next_shred_index, next_shred_index,
@ -346,11 +348,10 @@ pub(crate) mod tests {
let mut rng = rand::thread_rng(); let mut rng = rand::thread_rng();
let leader = Arc::new(Keypair::new()); let leader = Arc::new(Keypair::new());
let (slot, parent_slot, reference_tick, version) = (53084024, 53084023, 0, 0); let (slot, parent_slot, reference_tick, version) = (53084024, 53084023, 0, 0);
let shredder = let shredder = Shredder::new(slot, parent_slot, reference_tick, version).unwrap();
Shredder::new(slot, parent_slot, leader.clone(), reference_tick, version).unwrap();
let next_shred_index = rng.gen(); let next_shred_index = rng.gen();
let shred1 = new_rand_shred(&mut rng, next_shred_index, &shredder); let shred1 = new_rand_shred(&mut rng, next_shred_index, &shredder, &leader);
let shred2 = new_rand_shred(&mut rng, next_shred_index, &shredder); let shred2 = new_rand_shred(&mut rng, next_shred_index, &shredder, &leader);
let leader_schedule = |s| { let leader_schedule = |s| {
if s == slot { if s == slot {
Some(leader.pubkey()) Some(leader.pubkey())

30
ledger/src/blockstore.rs
View File

@ -1591,8 +1591,7 @@ impl Blockstore {
let mut remaining_ticks_in_slot = num_slots * ticks_per_slot - num_ticks_in_start_slot; let mut remaining_ticks_in_slot = num_slots * ticks_per_slot - num_ticks_in_start_slot;
let mut current_slot = start_slot; let mut current_slot = start_slot;
let mut shredder = let mut shredder = Shredder::new(current_slot, parent_slot, 0, version).unwrap();
Shredder::new(current_slot, parent_slot, keypair.clone(), 0, version).unwrap();
let mut all_shreds = vec![]; let mut all_shreds = vec![];
let mut slot_entries = vec![]; let mut slot_entries = vec![];
// Find all the entries for start_slot // Find all the entries for start_slot
@ -1611,13 +1610,12 @@ impl Blockstore {
} }
}; };
let (mut data_shreds, mut coding_shreds, _) = let (mut data_shreds, mut coding_shreds, _) =
shredder.entries_to_shreds(&current_entries, true, start_index); shredder.entries_to_shreds(keypair, &current_entries, true, start_index);
all_shreds.append(&mut data_shreds); all_shreds.append(&mut data_shreds);
all_shreds.append(&mut coding_shreds); all_shreds.append(&mut coding_shreds);
shredder = Shredder::new( shredder = Shredder::new(
current_slot, current_slot,
parent_slot, parent_slot,
keypair.clone(),
(ticks_per_slot - remaining_ticks_in_slot) as u8, (ticks_per_slot - remaining_ticks_in_slot) as u8,
version, version,
) )
@ -1632,7 +1630,7 @@ impl Blockstore {
if !slot_entries.is_empty() { if !slot_entries.is_empty() {
let (mut data_shreds, mut coding_shreds, _) = let (mut data_shreds, mut coding_shreds, _) =
shredder.entries_to_shreds(&slot_entries, is_full_slot, 0); shredder.entries_to_shreds(keypair, &slot_entries, is_full_slot, 0);
all_shreds.append(&mut data_shreds); all_shreds.append(&mut data_shreds);
all_shreds.append(&mut coding_shreds); all_shreds.append(&mut coding_shreds);
} }
@ -3530,8 +3528,10 @@ pub fn create_new_ledger(
let last_hash = entries.last().unwrap().hash; let last_hash = entries.last().unwrap().hash;
let version = solana_sdk::shred_version::version_from_hash(&last_hash); let version = solana_sdk::shred_version::version_from_hash(&last_hash);
let shredder = Shredder::new(0, 0, Arc::new(Keypair::new()), 0, version).unwrap(); let shredder = Shredder::new(0, 0, 0, version).unwrap();
let shreds = shredder.entries_to_shreds(&entries, true, 0).0; let shreds = shredder
.entries_to_shreds(&Keypair::new(), &entries, true, 0)
.0;
assert!(shreds.last().unwrap().last_in_slot()); assert!(shreds.last().unwrap().last_in_slot());
blockstore.insert_shreds(shreds, None, false)?; blockstore.insert_shreds(shreds, None, false)?;
@ -3712,9 +3712,9 @@ pub fn entries_to_test_shreds(
is_full_slot: bool, is_full_slot: bool,
version: u16, version: u16,
) -> Vec<Shred> { ) -> Vec<Shred> {
Shredder::new(slot, parent_slot, Arc::new(Keypair::new()), 0, version) Shredder::new(slot, parent_slot, 0, version)
.unwrap() .unwrap()
.entries_to_shreds(&entries, is_full_slot, 0) .entries_to_shreds(&Keypair::new(), &entries, is_full_slot, 0)
.0 .0
} }
@ -8007,8 +8007,9 @@ pub mod tests {
) -> (Vec<Shred>, Vec<Shred>, Arc<LeaderScheduleCache>) { ) -> (Vec<Shred>, Vec<Shred>, Arc<LeaderScheduleCache>) {
let entries = make_slot_entries_with_transactions(num_entries); let entries = make_slot_entries_with_transactions(num_entries);
let leader_keypair = Arc::new(Keypair::new()); let leader_keypair = Arc::new(Keypair::new());
let shredder = Shredder::new(slot, parent_slot, leader_keypair.clone(), 0, 0).unwrap(); let shredder = Shredder::new(slot, parent_slot, 0, 0).unwrap();
let (data_shreds, coding_shreds, _) = shredder.entries_to_shreds(&entries, true, 0); let (data_shreds, coding_shreds, _) =
shredder.entries_to_shreds(&leader_keypair, &entries, true, 0);
let genesis_config = create_genesis_config(2).genesis_config; let genesis_config = create_genesis_config(2).genesis_config;
let bank = Arc::new(Bank::new(&genesis_config)); let bank = Arc::new(Bank::new(&genesis_config));
@ -8062,9 +8063,10 @@ pub mod tests {
let entries1 = make_slot_entries_with_transactions(1); let entries1 = make_slot_entries_with_transactions(1);
let entries2 = make_slot_entries_with_transactions(1); let entries2 = make_slot_entries_with_transactions(1);
let leader_keypair = Arc::new(Keypair::new()); let leader_keypair = Arc::new(Keypair::new());
let shredder = Shredder::new(slot, 0, leader_keypair, 0, 0).unwrap(); let shredder = Shredder::new(slot, 0, 0, 0).unwrap();
let (shreds, _, _) = shredder.entries_to_shreds(&entries1, true, 0); let (shreds, _, _) = shredder.entries_to_shreds(&leader_keypair, &entries1, true, 0);
let (duplicate_shreds, _, _) = shredder.entries_to_shreds(&entries2, true, 0); let (duplicate_shreds, _, _) =
shredder.entries_to_shreds(&leader_keypair, &entries2, true, 0);
let shred = shreds[0].clone(); let shred = shreds[0].clone();
let duplicate_shred = duplicate_shreds[0].clone(); let duplicate_shred = duplicate_shreds[0].clone();
let non_duplicate_shred = shred.clone(); let non_duplicate_shred = shred.clone();

80
ledger/src/shred.rs
View File

@ -72,7 +72,7 @@ use solana_sdk::{
pubkey::Pubkey, pubkey::Pubkey,
signature::{Keypair, Signature, Signer}, signature::{Keypair, Signature, Signer},
}; };
use std::{mem::size_of, ops::Deref, sync::Arc}; use std::mem::size_of;
use thiserror::Error; use thiserror::Error;
@ -562,26 +562,18 @@ pub struct Shredder {
pub slot: Slot, pub slot: Slot,
pub parent_slot: Slot, pub parent_slot: Slot,
version: u16, version: u16,
keypair: Arc<Keypair>,
pub signing_coding_time: u128, pub signing_coding_time: u128,
reference_tick: u8, reference_tick: u8,
} }
impl Shredder { impl Shredder {
pub fn new( pub fn new(slot: Slot, parent_slot: Slot, reference_tick: u8, version: u16) -> Result<Self> {
slot: Slot,
parent_slot: Slot,
keypair: Arc<Keypair>,
reference_tick: u8,
version: u16,
) -> Result<Self> {
if slot < parent_slot || slot - parent_slot > u64::from(std::u16::MAX) { if slot < parent_slot || slot - parent_slot > u64::from(std::u16::MAX) {
Err(ShredError::SlotTooLow { slot, parent_slot }) Err(ShredError::SlotTooLow { slot, parent_slot })
} else { } else {
Ok(Self { Ok(Self {
slot, slot,
parent_slot, parent_slot,
keypair,
signing_coding_time: 0, signing_coding_time: 0,
reference_tick, reference_tick,
version, version,
@ -591,24 +583,22 @@ impl Shredder {
pub fn entries_to_shreds( pub fn entries_to_shreds(
&self, &self,
keypair: &Keypair,
entries: &[Entry], entries: &[Entry],
is_last_in_slot: bool, is_last_in_slot: bool,
next_shred_index: u32, next_shred_index: u32,
) -> (Vec<Shred>, Vec<Shred>, u32) { ) -> (Vec<Shred>, Vec<Shred>, u32) {
let mut stats = ProcessShredsStats::default(); let mut stats = ProcessShredsStats::default();
let (data_shreds, last_shred_index) = self.entries_to_data_shreds( let (data_shreds, last_shred_index) = self.entries_to_data_shreds(
keypair,
entries, entries,
is_last_in_slot, is_last_in_slot,
next_shred_index, next_shred_index,
next_shred_index, // fec_set_offset next_shred_index, // fec_set_offset
&mut stats, &mut stats,
); );
let coding_shreds = Self::data_shreds_to_coding_shreds( let coding_shreds =
self.keypair.deref(), Self::data_shreds_to_coding_shreds(keypair, &data_shreds, is_last_in_slot, &mut stats)
&data_shreds,
is_last_in_slot,
&mut stats,
)
.unwrap(); .unwrap();
(data_shreds, coding_shreds, last_shred_index) (data_shreds, coding_shreds, last_shred_index)
} }
@ -625,6 +615,7 @@ impl Shredder {
pub fn entries_to_data_shreds( pub fn entries_to_data_shreds(
&self, &self,
keypair: &Keypair,
entries: &[Entry], entries: &[Entry],
is_last_in_slot: bool, is_last_in_slot: bool,
next_shred_index: u32, next_shred_index: u32,
@ -659,7 +650,7 @@ impl Shredder {
self.version, self.version,
fec_set_index.unwrap(), fec_set_index.unwrap(),
); );
Shredder::sign_shred(self.keypair.deref(), &mut shred); Shredder::sign_shred(keypair, &mut shred);
shred shred
}; };
let data_shreds: Vec<Shred> = PAR_THREAD_POOL.with(|thread_pool| { let data_shreds: Vec<Shred> = PAR_THREAD_POOL.with(|thread_pool| {
@ -1109,7 +1100,7 @@ pub mod tests {
hash::{self, hash}, hash::{self, hash},
shred_version, system_transaction, shred_version, system_transaction,
}; };
use std::{collections::HashSet, convert::TryInto, iter::repeat_with}; use std::{collections::HashSet, convert::TryInto, iter::repeat_with, sync::Arc};
#[test] #[test]
fn test_shred_constants() { fn test_shred_constants() {
@ -1164,7 +1155,7 @@ pub mod tests {
// Test that parent cannot be > current slot // Test that parent cannot be > current slot
assert_matches!( assert_matches!(
Shredder::new(slot, slot + 1, keypair.clone(), 0, 0), Shredder::new(slot, slot + 1, 0, 0),
Err(ShredError::SlotTooLow { Err(ShredError::SlotTooLow {
slot: _, slot: _,
parent_slot: _, parent_slot: _,
@ -1172,14 +1163,14 @@ pub mod tests {
); );
// Test that slot - parent cannot be > u16 MAX // Test that slot - parent cannot be > u16 MAX
assert_matches!( assert_matches!(
Shredder::new(slot, slot - 1 - 0xffff, keypair.clone(), 0, 0), Shredder::new(slot, slot - 1 - 0xffff, 0, 0),
Err(ShredError::SlotTooLow { Err(ShredError::SlotTooLow {
slot: _, slot: _,
parent_slot: _, parent_slot: _,
}) })
); );
let parent_slot = slot - 5; let parent_slot = slot - 5;
let shredder = Shredder::new(slot, parent_slot, keypair.clone(), 0, 0).unwrap(); let shredder = Shredder::new(slot, parent_slot, 0, 0).unwrap();
let entries: Vec<_> = (0..5) let entries: Vec<_> = (0..5)
.map(|_| { .map(|_| {
let keypair0 = Keypair::new(); let keypair0 = Keypair::new();
@ -1199,7 +1190,7 @@ pub mod tests {
.max(num_expected_data_shreds as usize); .max(num_expected_data_shreds as usize);
let start_index = 0; let start_index = 0;
let (data_shreds, coding_shreds, next_index) = let (data_shreds, coding_shreds, next_index) =
shredder.entries_to_shreds(&entries, true, start_index); shredder.entries_to_shreds(&keypair, &entries, true, start_index);
assert_eq!(next_index as u64, num_expected_data_shreds); assert_eq!(next_index as u64, num_expected_data_shreds);
let mut data_shred_indexes = HashSet::new(); let mut data_shred_indexes = HashSet::new();
@ -1257,7 +1248,7 @@ pub mod tests {
let keypair = Arc::new(Keypair::new()); let keypair = Arc::new(Keypair::new());
let slot = 1; let slot = 1;
let parent_slot = 0; let parent_slot = 0;
let shredder = Shredder::new(slot, parent_slot, keypair, 0, 0).unwrap(); let shredder = Shredder::new(slot, parent_slot, 0, 0).unwrap();
let entries: Vec<_> = (0..5) let entries: Vec<_> = (0..5)
.map(|_| { .map(|_| {
let keypair0 = Keypair::new(); let keypair0 = Keypair::new();
@ -1268,7 +1259,7 @@ pub mod tests {
}) })
.collect(); .collect();
let data_shreds = shredder.entries_to_shreds(&entries, true, 0).0; let data_shreds = shredder.entries_to_shreds(&keypair, &entries, true, 0).0;
let deserialized_shred = let deserialized_shred =
Shred::new_from_serialized_shred(data_shreds.last().unwrap().payload.clone()).unwrap(); Shred::new_from_serialized_shred(data_shreds.last().unwrap().payload.clone()).unwrap();
@ -1280,7 +1271,7 @@ pub mod tests {
let keypair = Arc::new(Keypair::new()); let keypair = Arc::new(Keypair::new());
let slot = 1; let slot = 1;
let parent_slot = 0; let parent_slot = 0;
let shredder = Shredder::new(slot, parent_slot, keypair, 5, 0).unwrap(); let shredder = Shredder::new(slot, parent_slot, 5, 0).unwrap();
let entries: Vec<_> = (0..5) let entries: Vec<_> = (0..5)
.map(|_| { .map(|_| {
let keypair0 = Keypair::new(); let keypair0 = Keypair::new();
@ -1291,7 +1282,7 @@ pub mod tests {
}) })
.collect(); .collect();
let data_shreds = shredder.entries_to_shreds(&entries, true, 0).0; let data_shreds = shredder.entries_to_shreds(&keypair, &entries, true, 0).0;
data_shreds.iter().for_each(|s| { data_shreds.iter().for_each(|s| {
assert_eq!(s.reference_tick(), 5); assert_eq!(s.reference_tick(), 5);
assert_eq!(Shred::reference_tick_from_data(&s.payload), 5); assert_eq!(Shred::reference_tick_from_data(&s.payload), 5);
@ -1307,7 +1298,7 @@ pub mod tests {
let keypair = Arc::new(Keypair::new()); let keypair = Arc::new(Keypair::new());
let slot = 1; let slot = 1;
let parent_slot = 0; let parent_slot = 0;
let shredder = Shredder::new(slot, parent_slot, keypair, u8::max_value(), 0).unwrap(); let shredder = Shredder::new(slot, parent_slot, u8::max_value(), 0).unwrap();
let entries: Vec<_> = (0..5) let entries: Vec<_> = (0..5)
.map(|_| { .map(|_| {
let keypair0 = Keypair::new(); let keypair0 = Keypair::new();
@ -1318,7 +1309,7 @@ pub mod tests {
}) })
.collect(); .collect();
let data_shreds = shredder.entries_to_shreds(&entries, true, 0).0; let data_shreds = shredder.entries_to_shreds(&keypair, &entries, true, 0).0;
data_shreds.iter().for_each(|s| { data_shreds.iter().for_each(|s| {
assert_eq!(s.reference_tick(), SHRED_TICK_REFERENCE_MASK); assert_eq!(s.reference_tick(), SHRED_TICK_REFERENCE_MASK);
assert_eq!( assert_eq!(
@ -1337,7 +1328,7 @@ pub mod tests {
fn run_test_data_and_code_shredder(slot: Slot) { fn run_test_data_and_code_shredder(slot: Slot) {
let keypair = Arc::new(Keypair::new()); let keypair = Arc::new(Keypair::new());
let shredder = Shredder::new(slot, slot - 5, keypair.clone(), 0, 0).unwrap(); let shredder = Shredder::new(slot, slot - 5, 0, 0).unwrap();
// Create enough entries to make > 1 shred // Create enough entries to make > 1 shred
let payload_capacity = SIZE_OF_DATA_SHRED_PAYLOAD; let payload_capacity = SIZE_OF_DATA_SHRED_PAYLOAD;
let num_entries = max_ticks_per_n_shreds(1, Some(payload_capacity)) + 1; let num_entries = max_ticks_per_n_shreds(1, Some(payload_capacity)) + 1;
@ -1351,7 +1342,8 @@ pub mod tests {
}) })
.collect(); .collect();
let (data_shreds, coding_shreds, _) = shredder.entries_to_shreds(&entries, true, 0); let (data_shreds, coding_shreds, _) =
shredder.entries_to_shreds(&keypair, &entries, true, 0);
for (i, s) in data_shreds.iter().enumerate() { for (i, s) in data_shreds.iter().enumerate() {
verify_test_data_shred( verify_test_data_shred(
@ -1378,7 +1370,7 @@ pub mod tests {
fn run_test_recovery_and_reassembly(slot: Slot, is_last_in_slot: bool) { fn run_test_recovery_and_reassembly(slot: Slot, is_last_in_slot: bool) {
let keypair = Arc::new(Keypair::new()); let keypair = Arc::new(Keypair::new());
let shredder = Shredder::new(slot, slot - 5, keypair.clone(), 0, 0).unwrap(); let shredder = Shredder::new(slot, slot - 5, 0, 0).unwrap();
let keypair0 = Keypair::new(); let keypair0 = Keypair::new();
let keypair1 = Keypair::new(); let keypair1 = Keypair::new();
let tx0 = system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default()); let tx0 = system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default());
@ -1400,6 +1392,7 @@ pub mod tests {
let serialized_entries = bincode::serialize(&entries).unwrap(); let serialized_entries = bincode::serialize(&entries).unwrap();
let (data_shreds, coding_shreds, _) = shredder.entries_to_shreds( let (data_shreds, coding_shreds, _) = shredder.entries_to_shreds(
&keypair,
&entries, &entries,
is_last_in_slot, is_last_in_slot,
0, // next_shred_index 0, // next_shred_index
@ -1556,7 +1549,8 @@ pub mod tests {
// Test5: Try recovery/reassembly with non zero index full slot with 3 missing data shreds // Test5: Try recovery/reassembly with non zero index full slot with 3 missing data shreds
// and 2 missing coding shreds. Hint: should work // and 2 missing coding shreds. Hint: should work
let serialized_entries = bincode::serialize(&entries).unwrap(); let serialized_entries = bincode::serialize(&entries).unwrap();
let (data_shreds, coding_shreds, _) = shredder.entries_to_shreds(&entries, true, 25); let (data_shreds, coding_shreds, _) =
shredder.entries_to_shreds(&keypair, &entries, true, 25);
let num_coding_shreds = coding_shreds.len(); let num_coding_shreds = coding_shreds.len();
// We should have 10 shreds now // We should have 10 shreds now
assert_eq!(data_shreds.len(), num_data_shreds); assert_eq!(data_shreds.len(), num_data_shreds);
@ -1661,14 +1655,13 @@ pub mod tests {
let shredder = Shredder::new( let shredder = Shredder::new(
slot, slot,
slot - rng.gen_range(1, 27), // parent slot slot - rng.gen_range(1, 27), // parent slot
keypair,
0, // reference tick 0, // reference tick
rng.gen(), // version rng.gen(), // version
) )
.unwrap(); .unwrap();
let next_shred_index = rng.gen_range(1, 1024); let next_shred_index = rng.gen_range(1, 1024);
let (data_shreds, coding_shreds, _) = let (data_shreds, coding_shreds, _) =
shredder.entries_to_shreds(&[entry], is_last_in_slot, next_shred_index); shredder.entries_to_shreds(&keypair, &[entry], is_last_in_slot, next_shred_index);
let num_data_shreds = data_shreds.len(); let num_data_shreds = data_shreds.len();
let num_coding_shreds = coding_shreds.len(); let num_coding_shreds = coding_shreds.len();
let mut shreds = coding_shreds; let mut shreds = coding_shreds;
@ -1718,7 +1711,7 @@ pub mod tests {
let hash = hash(Hash::default().as_ref()); let hash = hash(Hash::default().as_ref());
let version = shred_version::version_from_hash(&hash); let version = shred_version::version_from_hash(&hash);
assert_ne!(version, 0); assert_ne!(version, 0);
let shredder = Shredder::new(0, 0, keypair, 0, version).unwrap(); let shredder = Shredder::new(0, 0, 0, version).unwrap();
let entries: Vec<_> = (0..5) let entries: Vec<_> = (0..5)
.map(|_| { .map(|_| {
let keypair0 = Keypair::new(); let keypair0 = Keypair::new();
@ -1730,7 +1723,7 @@ pub mod tests {
.collect(); .collect();
let (data_shreds, coding_shreds, _next_index) = let (data_shreds, coding_shreds, _next_index) =
shredder.entries_to_shreds(&entries, true, 0); shredder.entries_to_shreds(&keypair, &entries, true, 0);
assert!(!data_shreds assert!(!data_shreds
.iter() .iter()
.chain(coding_shreds.iter()) .chain(coding_shreds.iter())
@ -1766,7 +1759,7 @@ pub mod tests {
let hash = hash(Hash::default().as_ref()); let hash = hash(Hash::default().as_ref());
let version = shred_version::version_from_hash(&hash); let version = shred_version::version_from_hash(&hash);
assert_ne!(version, 0); assert_ne!(version, 0);
let shredder = Shredder::new(0, 0, keypair, 0, version).unwrap(); let shredder = Shredder::new(0, 0, 0, version).unwrap();
let entries: Vec<_> = (0..500) let entries: Vec<_> = (0..500)
.map(|_| { .map(|_| {
let keypair0 = Keypair::new(); let keypair0 = Keypair::new();
@ -1779,7 +1772,7 @@ pub mod tests {
let start_index = 0x12; let start_index = 0x12;
let (data_shreds, coding_shreds, _next_index) = let (data_shreds, coding_shreds, _next_index) =
shredder.entries_to_shreds(&entries, true, start_index); shredder.entries_to_shreds(&keypair, &entries, true, start_index);
let max_per_block = MAX_DATA_SHREDS_PER_FEC_BLOCK as usize; let max_per_block = MAX_DATA_SHREDS_PER_FEC_BLOCK as usize;
data_shreds.iter().enumerate().for_each(|(i, s)| { data_shreds.iter().enumerate().for_each(|(i, s)| {
@ -1802,7 +1795,7 @@ pub mod tests {
let hash = hash(Hash::default().as_ref()); let hash = hash(Hash::default().as_ref());
let version = shred_version::version_from_hash(&hash); let version = shred_version::version_from_hash(&hash);
assert_ne!(version, 0); assert_ne!(version, 0);
let shredder = Shredder::new(0, 0, keypair, 0, version).unwrap(); let shredder = Shredder::new(0, 0, 0, version).unwrap();
let entries: Vec<_> = (0..500) let entries: Vec<_> = (0..500)
.map(|_| { .map(|_| {
let keypair0 = Keypair::new(); let keypair0 = Keypair::new();
@ -1816,6 +1809,7 @@ pub mod tests {
let mut stats = ProcessShredsStats::default(); let mut stats = ProcessShredsStats::default();
let start_index = 0x12; let start_index = 0x12;
let (data_shreds, _next_index) = shredder.entries_to_data_shreds( let (data_shreds, _next_index) = shredder.entries_to_data_shreds(
&keypair,
&entries, &entries,
true, // is_last_in_slot true, // is_last_in_slot
start_index, start_index,
@ -1827,7 +1821,7 @@ pub mod tests {
(1..=MAX_DATA_SHREDS_PER_FEC_BLOCK as usize).for_each(|count| { (1..=MAX_DATA_SHREDS_PER_FEC_BLOCK as usize).for_each(|count| {
let coding_shreds = Shredder::data_shreds_to_coding_shreds( let coding_shreds = Shredder::data_shreds_to_coding_shreds(
shredder.keypair.deref(), &keypair,
&data_shreds[..count], &data_shreds[..count],
false, // is_last_in_slot false, // is_last_in_slot
&mut stats, &mut stats,
@ -1835,7 +1829,7 @@ pub mod tests {
.unwrap(); .unwrap();
assert_eq!(coding_shreds.len(), count); assert_eq!(coding_shreds.len(), count);
let coding_shreds = Shredder::data_shreds_to_coding_shreds( let coding_shreds = Shredder::data_shreds_to_coding_shreds(
shredder.keypair.deref(), &keypair,
&data_shreds[..count], &data_shreds[..count],
true, // is_last_in_slot true, // is_last_in_slot
&mut stats, &mut stats,
@ -1848,7 +1842,7 @@ pub mod tests {
}); });
let coding_shreds = Shredder::data_shreds_to_coding_shreds( let coding_shreds = Shredder::data_shreds_to_coding_shreds(
shredder.keypair.deref(), &keypair,
&data_shreds[..MAX_DATA_SHREDS_PER_FEC_BLOCK as usize + 1], &data_shreds[..MAX_DATA_SHREDS_PER_FEC_BLOCK as usize + 1],
false, // is_last_in_slot false, // is_last_in_slot
&mut stats, &mut stats,
@ -1859,7 +1853,7 @@ pub mod tests {
MAX_DATA_SHREDS_PER_FEC_BLOCK as usize + 1 MAX_DATA_SHREDS_PER_FEC_BLOCK as usize + 1
); );
let coding_shreds = Shredder::data_shreds_to_coding_shreds( let coding_shreds = Shredder::data_shreds_to_coding_shreds(
shredder.keypair.deref(), &keypair,
&data_shreds[..MAX_DATA_SHREDS_PER_FEC_BLOCK as usize + 1], &data_shreds[..MAX_DATA_SHREDS_PER_FEC_BLOCK as usize + 1],
true, // is_last_in_slot true, // is_last_in_slot
&mut stats, &mut stats,

9
ledger/tests/shred.rs
View File

@ -22,7 +22,7 @@ type IndexShredsMap = BTreeMap<u32, Vec<Shred>>;
fn test_multi_fec_block_coding() { fn test_multi_fec_block_coding() {
let keypair = Arc::new(Keypair::new()); let keypair = Arc::new(Keypair::new());
let slot = 0x1234_5678_9abc_def0; let slot = 0x1234_5678_9abc_def0;
let shredder = Shredder::new(slot, slot - 5, keypair.clone(), 0, 0).unwrap(); let shredder = Shredder::new(slot, slot - 5, 0, 0).unwrap();
let num_fec_sets = 100; let num_fec_sets = 100;
let num_data_shreds = (MAX_DATA_SHREDS_PER_FEC_BLOCK * num_fec_sets) as usize; let num_data_shreds = (MAX_DATA_SHREDS_PER_FEC_BLOCK * num_fec_sets) as usize;
let keypair0 = Keypair::new(); let keypair0 = Keypair::new();
@ -46,7 +46,8 @@ fn test_multi_fec_block_coding() {
.collect(); .collect();
let serialized_entries = bincode::serialize(&entries).unwrap(); let serialized_entries = bincode::serialize(&entries).unwrap();
let (data_shreds, coding_shreds, next_index) = shredder.entries_to_shreds(&entries, true, 0); let (data_shreds, coding_shreds, next_index) =
shredder.entries_to_shreds(&keypair, &entries, true, 0);
assert_eq!(next_index as usize, num_data_shreds); assert_eq!(next_index as usize, num_data_shreds);
assert_eq!(data_shreds.len(), num_data_shreds); assert_eq!(data_shreds.len(), num_data_shreds);
assert_eq!(coding_shreds.len(), num_data_shreds); assert_eq!(coding_shreds.len(), num_data_shreds);
@ -190,7 +191,7 @@ fn setup_different_sized_fec_blocks(
parent_slot: Slot, parent_slot: Slot,
keypair: Arc<Keypair>, keypair: Arc<Keypair>,
) -> (IndexShredsMap, IndexShredsMap, usize) { ) -> (IndexShredsMap, IndexShredsMap, usize) {
let shredder = Shredder::new(slot, parent_slot, keypair, 0, 0).unwrap(); let shredder = Shredder::new(slot, parent_slot, 0, 0).unwrap();
let keypair0 = Keypair::new(); let keypair0 = Keypair::new();
let keypair1 = Keypair::new(); let keypair1 = Keypair::new();
let tx0 = system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default()); let tx0 = system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default());
@ -227,7 +228,7 @@ fn setup_different_sized_fec_blocks(
for i in 0..2 { for i in 0..2 {
let is_last = i == 1; let is_last = i == 1;
let (data_shreds, coding_shreds, new_next_index) = let (data_shreds, coding_shreds, new_next_index) =
shredder.entries_to_shreds(&entries, is_last, next_index); shredder.entries_to_shreds(&keypair, &entries, is_last, next_index);
for shred in &data_shreds { for shred in &data_shreds {
if (shred.index() as usize) == total_num_data_shreds - 1 { if (shred.index() as usize) == total_num_data_shreds - 1 {
assert!(shred.data_complete()); assert!(shred.data_complete());