gaspersic/solana
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Integrate data shreds (#5541)

Browse Source 
* Insert data shreds in blocktree and database

* Integrate data shreds with rest of the code base

* address review comments, and some clippy fixes

* Fixes to some tests

* more test fixes

* ignore some local cluster tests

* ignore replicator local cluster tests
This commit is contained in:
Pankaj Garg
2019-08-20 17:16:06 -07:00
committed by GitHub
parent f4534ef12d
commit 4798e7fa73
28 changed files with 1325 additions and 612 deletions
Download Patch File Download Diff File Expand all files Collapse all files

244
core/src/window_service.rs
View File

@ -4,17 +4,14 @@
use crate::blocktree::Blocktree;
use crate::cluster_info::ClusterInfo;
use crate::leader_schedule_cache::LeaderScheduleCache;
use crate::packet::{Blob, SharedBlob};
use crate::repair_service::{RepairService, RepairStrategy};
use crate::result::{Error, Result};
use crate::service::Service;
use crate::streamer::{BlobReceiver, BlobSender};
use rayon::prelude::*;
use rayon::ThreadPool;
use crate::shred::Shred;
use crate::streamer::{PacketReceiver, PacketSender};
use solana_metrics::{inc_new_counter_debug, inc_new_counter_error};
use solana_runtime::bank::Bank;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::signature::Signable;
use solana_sdk::timing::duration_as_ms;
use std::net::UdpSocket;
use std::sync::atomic::{AtomicBool, Ordering};
@ -25,114 +22,99 @@ use std::time::{Duration, Instant};
pub const NUM_THREADS: u32 = 10;
fn retransmit_blobs(blobs: &[SharedBlob], retransmit: &BlobSender, id: &Pubkey) -> Result<()> {
let mut retransmit_queue: Vec<SharedBlob> = Vec::new();
for blob in blobs {
let mut blob_guard = blob.write().unwrap();
// Don't add blobs generated by this node to the retransmit queue
if blob_guard.id() != *id && !blob_guard.is_coding() {
//let mut w_blob = blob.write().unwrap();
blob_guard.meta.forward = blob_guard.should_forward();
blob_guard.set_forwarded(false);
retransmit_queue.push(blob.clone());
}
}
if !retransmit_queue.is_empty() {
inc_new_counter_debug!("streamer-recv_window-retransmit", retransmit_queue.len());
retransmit.send(retransmit_queue)?;
}
Ok(())
}
/// Process a blob: Add blob to the ledger window.
pub fn process_blobs(blobs: &[SharedBlob], blocktree: &Arc<Blocktree>) -> Result<()> {
// make an iterator for insert_data_blobs()
//let blobs: Vec<_> = blobs.iter().map(move |blob| blob.read().unwrap()).collect();
blocktree.write_shared_blobs(
blobs
.iter()
.filter(|blob| !blob.read().unwrap().is_coding()),
)?;
blocktree
.put_shared_coding_blobs(blobs.iter().filter(|blob| blob.read().unwrap().is_coding()))?;
Ok(())
pub fn process_shreds(shreds: &[Shred], blocktree: &Arc<Blocktree>) -> Result<()> {
blocktree.insert_shreds(shreds)
}
/// drop blobs that are from myself or not from the correct leader for the
/// blob's slot
pub fn should_retransmit_and_persist(
blob: &Blob,
shred: &Shred,
bank: Option<Arc<Bank>>,
leader_schedule_cache: &Arc<LeaderScheduleCache>,
my_pubkey: &Pubkey,
) -> bool {
let slot_leader_pubkey = match bank {
None => leader_schedule_cache.slot_leader_at(blob.slot(), None),
Some(bank) => leader_schedule_cache.slot_leader_at(blob.slot(), Some(&bank)),
None => leader_schedule_cache.slot_leader_at(shred.slot(), None),
Some(bank) => leader_schedule_cache.slot_leader_at(shred.slot(), Some(&bank)),
};
if !blob.verify() {
inc_new_counter_debug!("streamer-recv_window-invalid_signature", 1);
false
} else if blob.id() == *my_pubkey {
inc_new_counter_debug!("streamer-recv_window-circular_transmission", 1);
false
} else if slot_leader_pubkey == None {
if let Some(leader_id) = slot_leader_pubkey {
if leader_id == *my_pubkey {
inc_new_counter_debug!("streamer-recv_window-circular_transmission", 1);
false
} else if !shred.verify(&leader_id) {
inc_new_counter_debug!("streamer-recv_window-invalid_signature", 1);
false
} else {
true
}
} else {
inc_new_counter_debug!("streamer-recv_window-unknown_leader", 1);
false
} else if slot_leader_pubkey != Some(blob.id()) {
inc_new_counter_debug!("streamer-recv_window-wrong_leader", 1);
false
} else {
// At this point, slot_leader_id == blob.id() && blob.id() != *my_id, so
// the blob is valid to process
true
}
}
fn recv_window<F>(
blocktree: &Arc<Blocktree>,
my_pubkey: &Pubkey,
r: &BlobReceiver,
retransmit: &BlobSender,
blob_filter: F,
thread_pool: &ThreadPool,
r: &PacketReceiver,
retransmit: &PacketSender,
shred_filter: F,
) -> Result<()>
where
F: Fn(&Blob) -> bool,
F: Fn(&Shred) -> bool,
F: Sync,
{
let timer = Duration::from_millis(200);
let mut blobs = r.recv_timeout(timer)?;
let mut packets = r.recv_timeout(timer)?;
while let Ok(mut blob) = r.try_recv() {
blobs.append(&mut blob)
while let Ok(mut more_packets) = r.try_recv() {
packets.packets.append(&mut more_packets.packets)
}
let now = Instant::now();
inc_new_counter_debug!("streamer-recv_window-recv", blobs.len());
inc_new_counter_debug!("streamer-recv_window-recv", packets.packets.len());
let blobs: Vec<_> = thread_pool.install(|| {
blobs
.into_par_iter()
.filter(|b| blob_filter(&b.read().unwrap()))
.collect()
});
let mut shreds = vec![];
let mut discards = vec![];
for (i, packet) in packets.packets.iter_mut().enumerate() {
if let Ok(s) = bincode::deserialize(&packet.data) {
let shred: Shred = s;
if shred_filter(&shred) {
packet.meta.slot = shred.slot();
packet.meta.seed = shred.seed();
shreds.push(shred);
} else {
discards.push(i);
}
} else {
discards.push(i);
}
}
match retransmit_blobs(&blobs, retransmit, my_pubkey) {
Ok(_) => Ok(()),
Err(Error::SendError) => Ok(()),
Err(e) => Err(e),
}?;
for i in discards.into_iter().rev() {
packets.packets.remove(i);
}
trace!("{} num blobs received: {}", my_pubkey, blobs.len());
process_blobs(&blobs, blocktree)?;
trace!("{:?} shreds from packets", shreds.len());
trace!(
"{} num shreds received: {}",
my_pubkey,
packets.packets.len()
);
if !packets.packets.is_empty() {
match retransmit.send(packets) {
Ok(_) => Ok(()),
Err(e) => Err(e),
}?;
}
blocktree.insert_shreds(&shreds)?;
info!(
"Elapsed processing time in recv_window(): {}",
duration_as_ms(&now.elapsed())
);
@ -168,16 +150,16 @@ impl WindowService {
pub fn new<F>(
blocktree: Arc<Blocktree>,
cluster_info: Arc<RwLock<ClusterInfo>>,
r: BlobReceiver,
retransmit: BlobSender,
r: PacketReceiver,
retransmit: PacketSender,
repair_socket: Arc<UdpSocket>,
exit: &Arc<AtomicBool>,
repair_strategy: RepairStrategy,
blob_filter: F,
shred_filter: F,
) -> WindowService
where
F: 'static
+ Fn(&Pubkey, &Blob, Option<Arc<Bank>>) -> bool
+ Fn(&Pubkey, &Shred, Option<Arc<Bank>>) -> bool
+ std::marker::Send
+ std::marker::Sync,
{
@ -195,7 +177,7 @@ impl WindowService {
repair_strategy,
);
let exit = exit.clone();
let blob_filter = Arc::new(blob_filter);
let shred_filter = Arc::new(shred_filter);
let bank_forks = bank_forks.clone();
let t_window = Builder::new()
.name("solana-window".to_string())
@ -205,10 +187,6 @@ impl WindowService {
let _exit = Finalizer::new(exit.clone());
let id = cluster_info.read().unwrap().id();
trace!("{}: RECV_WINDOW started", id);
let thread_pool = rayon::ThreadPoolBuilder::new()
.num_threads(sys_info::cpu_num().unwrap_or(NUM_THREADS) as usize)
.build()
.unwrap();
let mut now = Instant::now();
loop {
if exit.load(Ordering::Relaxed) {
@ -220,16 +198,15 @@ impl WindowService {
&id,
&r,
&retransmit,
|blob| {
blob_filter(
|shred| {
shred_filter(
&id,
blob,
shred,
bank_forks
.as_ref()
.map(|bank_forks| bank_forks.read().unwrap().working_bank()),
)
},
&thread_pool,
) {
match e {
Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
@ -272,12 +249,14 @@ mod test {
use super::*;
use crate::bank_forks::BankForks;
use crate::blocktree::{get_tmp_ledger_path, Blocktree};
use crate::broadcast_stage::broadcast_utils::entries_to_shreds;
use crate::cluster_info::{ClusterInfo, Node};
use crate::entry::{make_consecutive_blobs, make_tiny_test_entries, Entry, EntrySlice};
use crate::entry::{make_consecutive_blobs, make_tiny_test_entries, Entry};
use crate::genesis_utils::create_genesis_block_with_leader;
use crate::packet::index_blobs;
use crate::recycler::Recycler;
use crate::service::Service;
use crate::streamer::{blob_receiver, responder};
use crate::shred::Shredder;
use crate::streamer::{receiver, responder};
use solana_runtime::epoch_schedule::MINIMUM_SLOTS_PER_EPOCH;
use solana_sdk::hash::Hash;
use solana_sdk::signature::{Keypair, KeypairUtil};
@ -288,18 +267,27 @@ mod test {
use std::sync::{Arc, RwLock};
use std::time::Duration;
fn local_entries_to_shred(entries: Vec<Entry>, keypair: &Arc<Keypair>) -> Vec<Shred> {
let mut shredder =
Shredder::new(0, Some(0), 0.0, keypair, 0).expect("Failed to create entry shredder");
entries_to_shreds(vec![entries], 0, 0, &mut shredder);
shredder
.shreds
.iter()
.map(|s| bincode::deserialize(s).unwrap())
.collect()
}
#[test]
fn test_process_blob() {
let blocktree_path = get_tmp_ledger_path!();
let blocktree = Arc::new(Blocktree::open(&blocktree_path).unwrap());
let num_entries = 10;
let original_entries = make_tiny_test_entries(num_entries);
let shared_blobs = original_entries.clone().to_shared_blobs();
let shreds = local_entries_to_shred(original_entries.clone(), &Arc::new(Keypair::new()));
index_blobs(&shared_blobs, &Pubkey::new_rand(), 0, 0, 0);
for blob in shared_blobs.into_iter().rev() {
process_blobs(&[blob], &blocktree).expect("Expect successful processing of blob");
for shred in shreds.into_iter().rev() {
process_shreds(&[shred], &blocktree).expect("Expect successful processing of blob");
}
assert_eq!(
@ -322,39 +310,40 @@ mod test {
let cache = Arc::new(LeaderScheduleCache::new_from_bank(&bank));
let entry = Entry::default();
let mut blob = entry.to_blob();
blob.set_id(&leader_pubkey);
blob.sign(&leader_keypair);
let mut shreds = local_entries_to_shred(vec![entry], &Arc::new(leader_keypair));
// with a Bank for slot 0, blob continues
assert_eq!(
should_retransmit_and_persist(&blob, Some(bank.clone()), &cache, &me_id),
should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id),
true
);
// set the blob to have come from the wrong leader
blob.set_id(&Pubkey::new_rand());
assert_eq!(
should_retransmit_and_persist(&blob, Some(bank.clone()), &cache, &me_id),
false
);
/*
assert_eq!(
should_retransmit_and_persist(&shreds[0], Some(bank.clone()), &cache, &me_id),
false
);
*/
// with a Bank and no idea who leader is, blob gets thrown out
blob.set_slot(MINIMUM_SLOTS_PER_EPOCH as u64 * 3);
shreds[0].set_slot(MINIMUM_SLOTS_PER_EPOCH as u64 * 3);
assert_eq!(
should_retransmit_and_persist(&blob, Some(bank), &cache, &me_id),
should_retransmit_and_persist(&shreds[0], Some(bank), &cache, &me_id),
false
);
// if the blob came back from me, it doesn't continue, whether or not I have a bank
blob.set_id(&me_id);
assert_eq!(
should_retransmit_and_persist(&blob, None, &cache, &me_id),
false
);
/*
assert_eq!(
should_retransmit_and_persist(&shreds[0], None, &cache, &me_id),
false
);
*/
}
#[test]
#[ignore]
pub fn window_send_test() {
solana_logger::setup();
// setup a leader whose id is used to generates blobs and a validator
@ -367,7 +356,13 @@ mod test {
let subs = Arc::new(RwLock::new(cluster_info_me));
let (s_reader, r_reader) = channel();
let t_receiver = blob_receiver(Arc::new(leader_node.sockets.gossip), &exit, s_reader);
let t_receiver = receiver(
Arc::new(leader_node.sockets.gossip),
&exit,
s_reader,
Recycler::default(),
"window_send_test",
);
let (s_retransmit, r_retransmit) = channel();
let blocktree_path = get_tmp_ledger_path!();
let (blocktree, _, completed_slots_receiver) = Blocktree::open_with_signal(&blocktree_path)
@ -424,7 +419,7 @@ mod test {
loop {
assert!(num_attempts != max_attempts);
while let Ok(mut nq) = r_retransmit.recv_timeout(Duration::from_millis(500)) {
q.append(&mut nq);
q.append(&mut nq.packets);
}
if q.len() == 10 {
break;
@ -441,6 +436,7 @@ mod test {
}
#[test]
#[ignore]
pub fn window_send_leader_test2() {
solana_logger::setup();
// setup a leader whose id is used to generates blobs and a validator
@ -453,7 +449,13 @@ mod test {
let subs = Arc::new(RwLock::new(cluster_info_me));
let (s_reader, r_reader) = channel();
let t_receiver = blob_receiver(Arc::new(leader_node.sockets.gossip), &exit, s_reader);
let t_receiver = receiver(
Arc::new(leader_node.sockets.gossip),
&exit,
s_reader,
Recycler::default(),
"window_send_leader_test2",
);
let (s_retransmit, r_retransmit) = channel();
let blocktree_path = get_tmp_ledger_path!();
let (blocktree, _, completed_slots_receiver) = Blocktree::open_with_signal(&blocktree_path)
@ -502,8 +504,8 @@ mod test {
t_responder
};
let mut q = Vec::new();
while let Ok(mut nq) = r_retransmit.recv_timeout(Duration::from_millis(500)) {
q.append(&mut nq);
while let Ok(mut nq) = r_retransmit.recv_timeout(Duration::from_millis(5000)) {
q.append(&mut nq.packets);
}
assert!(q.len() > 10);
exit.store(true, Ordering::Relaxed);