Randomize avalanche broadcast peer table for each blob (#4529)
* fix clippy warnings
This commit is contained in:
Pankaj Garg
GitHub
@ -1,6 +1,6 @@
|
||||
use rand::SeedableRng;
|
||||
use rand_chacha::ChaChaRng;
|
||||
use rayon::iter::{IntoParallelIterator, ParallelIterator};
|
||||
use rayon::iter::ParallelIterator;
|
||||
use rayon::prelude::*;
|
||||
use solana::cluster_info::{compute_retransmit_peers, ClusterInfo};
|
||||
use solana::contact_info::ContactInfo;
|
||||
@ -9,11 +9,11 @@ use std::collections::{HashMap, HashSet};
|
||||
use std::sync::mpsc::channel;
|
||||
use std::sync::mpsc::TryRecvError;
|
||||
use std::sync::mpsc::{Receiver, Sender};
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::Instant;
|
||||
|
||||
type Nodes = HashMap<Pubkey, (HashSet<i32>, Receiver<(i32, bool)>)>;
|
||||
type Nodes = HashMap<Pubkey, (bool, HashSet<i32>, Receiver<(i32, bool)>)>;
|
||||
|
||||
fn num_threads() -> usize {
|
||||
sys_info::cpu_num().unwrap_or(10) as usize
|
||||
@ -23,11 +23,48 @@ fn num_threads() -> usize {
|
||||
fn find_insert_blob(id: &Pubkey, blob: i32, batches: &mut [Nodes]) {
|
||||
batches.par_iter_mut().for_each(|batch| {
|
||||
if batch.contains_key(id) {
|
||||
let _ = batch.get_mut(id).unwrap().0.insert(blob);
|
||||
let _ = batch.get_mut(id).unwrap().1.insert(blob);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn retransmit(
|
||||
mut shuffled_nodes: Vec<ContactInfo>,
|
||||
senders: &HashMap<Pubkey, Sender<(i32, bool)>>,
|
||||
cluster: &ClusterInfo,
|
||||
fanout: usize,
|
||||
blob: i32,
|
||||
retransmit: bool,
|
||||
) -> i32 {
|
||||
let mut seed = [0; 32];
|
||||
let mut my_index = 0;
|
||||
let mut index = 0;
|
||||
shuffled_nodes.retain(|c| {
|
||||
if c.id == cluster.id() {
|
||||
my_index = index;
|
||||
false
|
||||
} else {
|
||||
index += 1;
|
||||
true
|
||||
}
|
||||
});
|
||||
seed[0..4].copy_from_slice(&blob.to_le_bytes());
|
||||
let (neighbors, children) = compute_retransmit_peers(fanout, my_index, shuffled_nodes);
|
||||
children.iter().for_each(|p| {
|
||||
let s = senders.get(&p.id).unwrap();
|
||||
let _ = s.send((blob, retransmit));
|
||||
});
|
||||
|
||||
if retransmit {
|
||||
neighbors.iter().for_each(|p| {
|
||||
let s = senders.get(&p.id).unwrap();
|
||||
let _ = s.send((blob, false));
|
||||
});
|
||||
}
|
||||
|
||||
blob
|
||||
}
|
||||
|
||||
fn run_simulation(stakes: &[u64], fanout: usize) {
|
||||
let num_threads = num_threads();
|
||||
// set timeout to 5 minutes
|
||||
@ -50,7 +87,7 @@ fn run_simulation(stakes: &[u64], fanout: usize) {
|
||||
batches
|
||||
.get_mut(0)
|
||||
.unwrap()
|
||||
.insert(leader_info.id, (HashSet::new(), r));
|
||||
.insert(leader_info.id, (false, HashSet::new(), r));
|
||||
let range: Vec<_> = (1..=stakes.len()).collect();
|
||||
let chunk_size = (stakes.len() + num_threads - 1) / num_threads;
|
||||
range.chunks(chunk_size).for_each(|chunk| {
|
||||
@ -64,99 +101,77 @@ fn run_simulation(stakes: &[u64], fanout: usize) {
|
||||
batches
|
||||
.get_mut(batch_ix)
|
||||
.unwrap()
|
||||
.insert(node.id, (HashSet::new(), r));
|
||||
.insert(node.id, (false, HashSet::new(), r));
|
||||
senders.lock().unwrap().insert(node.id, s);
|
||||
})
|
||||
});
|
||||
let c_info = cluster_info.clone();
|
||||
|
||||
// create some "blobs".
|
||||
let blobs: Vec<(_, _)> = (0..100).into_par_iter().map(|i| (i as i32, true)).collect();
|
||||
|
||||
// pretend to broadcast from leader - cluster_info::create_broadcast_orders
|
||||
let mut broadcast_table =
|
||||
cluster_info.sorted_tvu_peers(Some(&staked_nodes), ChaChaRng::from_seed([0x5a; 32]));
|
||||
broadcast_table.truncate(fanout);
|
||||
let orders = ClusterInfo::create_broadcast_orders(false, &blobs, &broadcast_table);
|
||||
|
||||
// send blobs to layer 1 nodes
|
||||
orders.iter().for_each(|(b, vc)| {
|
||||
vc.iter().for_each(|c| {
|
||||
find_insert_blob(&c.id, b.0, &mut batches);
|
||||
let blobs_len = 100;
|
||||
let shuffled_peers: Vec<Vec<ContactInfo>> = (0..blobs_len as i32)
|
||||
.map(|i| {
|
||||
let mut seed = [0; 32];
|
||||
seed[0..4].copy_from_slice(&i.to_le_bytes());
|
||||
let (_, peers) = cluster_info
|
||||
.shuffle_peers_and_index(Some(&staked_nodes), ChaChaRng::from_seed(seed));
|
||||
peers
|
||||
})
|
||||
.collect();
|
||||
|
||||
// create some "blobs".
|
||||
(0..blobs_len).into_iter().for_each(|i| {
|
||||
let broadcast_table = &shuffled_peers[i];
|
||||
find_insert_blob(&broadcast_table[0].id, i as i32, &mut batches);
|
||||
});
|
||||
|
||||
assert!(!batches.is_empty());
|
||||
|
||||
// start avalanche simulation
|
||||
let now = Instant::now();
|
||||
batches.par_iter_mut().for_each(|batch| {
|
||||
let mut cluster = c_info.clone();
|
||||
let batch_size = batch.len();
|
||||
let mut remaining = batch_size;
|
||||
let mut remaining = batch.len();
|
||||
let senders: HashMap<_, _> = senders.lock().unwrap().clone();
|
||||
// A map that holds neighbors and children senders for a given node
|
||||
let mut mapped_peers: HashMap<
|
||||
Pubkey,
|
||||
(Vec<Sender<(i32, bool)>>, Vec<Sender<(i32, bool)>>),
|
||||
> = HashMap::new();
|
||||
while remaining > 0 {
|
||||
for (id, (recv, r)) in batch.iter_mut() {
|
||||
for (id, (layer1_done, recv, r)) in batch.iter_mut() {
|
||||
assert!(
|
||||
now.elapsed().as_secs() < timeout,
|
||||
"Timed out with {:?} remaining nodes",
|
||||
remaining
|
||||
);
|
||||
cluster.gossip.set_self(&*id);
|
||||
if !mapped_peers.contains_key(id) {
|
||||
let (neighbors, children) = compute_retransmit_peers(
|
||||
Some(&staked_nodes),
|
||||
&Arc::new(RwLock::new(cluster.clone())),
|
||||
fanout,
|
||||
ChaChaRng::from_seed([0x5a; 32]),
|
||||
);
|
||||
let vec_children: Vec<_> = children
|
||||
.iter()
|
||||
.map(|p| {
|
||||
let s = senders.get(&p.id).unwrap();
|
||||
recv.iter().for_each(|i| {
|
||||
let _ = s.send((*i, true));
|
||||
});
|
||||
s.clone()
|
||||
})
|
||||
.collect();
|
||||
|
||||
let vec_neighbors: Vec<_> = neighbors
|
||||
.iter()
|
||||
.map(|p| {
|
||||
let s = senders.get(&p.id).unwrap();
|
||||
recv.iter().for_each(|i| {
|
||||
let _ = s.send((*i, false));
|
||||
});
|
||||
s.clone()
|
||||
})
|
||||
.collect();
|
||||
mapped_peers.insert(*id, (vec_neighbors, vec_children));
|
||||
if !*layer1_done {
|
||||
recv.iter().for_each(|i| {
|
||||
retransmit(
|
||||
shuffled_peers[*i as usize].clone(),
|
||||
&senders,
|
||||
&cluster,
|
||||
fanout,
|
||||
*i,
|
||||
true,
|
||||
);
|
||||
});
|
||||
*layer1_done = true;
|
||||
}
|
||||
let (vec_neighbors, vec_children) = mapped_peers.get(id).unwrap();
|
||||
|
||||
//send and recv
|
||||
if recv.len() < blobs.len() {
|
||||
if recv.len() < blobs_len {
|
||||
loop {
|
||||
match r.try_recv() {
|
||||
Ok((data, retransmit)) => {
|
||||
Ok((data, retx)) => {
|
||||
if recv.insert(data) {
|
||||
vec_children.iter().for_each(|s| {
|
||||
let _ = s.send((data, retransmit));
|
||||
});
|
||||
if retransmit {
|
||||
vec_neighbors.iter().for_each(|s| {
|
||||
let _ = s.send((data, false));
|
||||
})
|
||||
}
|
||||
if recv.len() == blobs.len() {
|
||||
remaining -= 1;
|
||||
break;
|
||||
}
|
||||
let _ = retransmit(
|
||||
shuffled_peers[data as usize].clone(),
|
||||
&senders,
|
||||
&cluster,
|
||||
fanout,
|
||||
data,
|
||||
retx,
|
||||
);
|
||||
}
|
||||
if recv.len() == blobs_len {
|
||||
remaining -= 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(TryRecvError::Disconnected) => break,
|
||||
|
||||
Reference in New Issue
Block a user