Rename Packets to PacketBatch (#21794)
This commit is contained in:
Justin Starry
GitHub
@ -13,13 +13,13 @@ pub const PACKETS_PER_BATCH: usize = 128;
|
||||
pub const NUM_RCVMMSGS: usize = 128;
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct Packets {
|
||||
pub struct PacketBatch {
|
||||
pub packets: PinnedVec<Packet>,
|
||||
}
|
||||
|
||||
pub type PacketsRecycler = Recycler<PinnedVec<Packet>>;
|
||||
pub type PacketBatchRecycler = Recycler<PinnedVec<Packet>>;
|
||||
|
||||
impl Packets {
|
||||
impl PacketBatch {
|
||||
pub fn new(packets: Vec<Packet>) -> Self {
|
||||
let packets = PinnedVec::from_vec(packets);
|
||||
Self { packets }
|
||||
@ -27,48 +27,52 @@ impl Packets {
|
||||
|
||||
pub fn with_capacity(capacity: usize) -> Self {
|
||||
let packets = PinnedVec::with_capacity(capacity);
|
||||
Packets { packets }
|
||||
PacketBatch { packets }
|
||||
}
|
||||
|
||||
pub fn new_unpinned_with_recycler(
|
||||
recycler: PacketsRecycler,
|
||||
recycler: PacketBatchRecycler,
|
||||
size: usize,
|
||||
name: &'static str,
|
||||
) -> Self {
|
||||
let mut packets = recycler.allocate(name);
|
||||
packets.reserve(size);
|
||||
Packets { packets }
|
||||
PacketBatch { packets }
|
||||
}
|
||||
|
||||
pub fn new_with_recycler(recycler: PacketsRecycler, size: usize, name: &'static str) -> Self {
|
||||
pub fn new_with_recycler(
|
||||
recycler: PacketBatchRecycler,
|
||||
size: usize,
|
||||
name: &'static str,
|
||||
) -> Self {
|
||||
let mut packets = recycler.allocate(name);
|
||||
packets.reserve_and_pin(size);
|
||||
Packets { packets }
|
||||
PacketBatch { packets }
|
||||
}
|
||||
|
||||
pub fn new_with_recycler_data(
|
||||
recycler: &PacketsRecycler,
|
||||
recycler: &PacketBatchRecycler,
|
||||
name: &'static str,
|
||||
mut packets: Vec<Packet>,
|
||||
) -> Self {
|
||||
let mut vec = Self::new_with_recycler(recycler.clone(), packets.len(), name);
|
||||
vec.packets.append(&mut packets);
|
||||
vec
|
||||
let mut batch = Self::new_with_recycler(recycler.clone(), packets.len(), name);
|
||||
batch.packets.append(&mut packets);
|
||||
batch
|
||||
}
|
||||
|
||||
pub fn new_unpinned_with_recycler_data(
|
||||
recycler: &PacketsRecycler,
|
||||
recycler: &PacketBatchRecycler,
|
||||
name: &'static str,
|
||||
mut packets: Vec<Packet>,
|
||||
) -> Self {
|
||||
let mut vec = Self::new_unpinned_with_recycler(recycler.clone(), packets.len(), name);
|
||||
vec.packets.append(&mut packets);
|
||||
vec
|
||||
let mut batch = Self::new_unpinned_with_recycler(recycler.clone(), packets.len(), name);
|
||||
batch.packets.append(&mut packets);
|
||||
batch
|
||||
}
|
||||
|
||||
pub fn set_addr(&mut self, addr: &SocketAddr) {
|
||||
for m in self.packets.iter_mut() {
|
||||
m.meta.set_addr(addr);
|
||||
for p in self.packets.iter_mut() {
|
||||
p.meta.set_addr(addr);
|
||||
}
|
||||
}
|
||||
|
||||
@ -77,32 +81,32 @@ impl Packets {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_packets_chunked<T: Serialize>(xs: &[T], chunks: usize) -> Vec<Packets> {
|
||||
pub fn to_packet_batches<T: Serialize>(xs: &[T], chunks: usize) -> Vec<PacketBatch> {
|
||||
let mut out = vec![];
|
||||
for x in xs.chunks(chunks) {
|
||||
let mut p = Packets::with_capacity(x.len());
|
||||
p.packets.resize(x.len(), Packet::default());
|
||||
for (i, o) in x.iter().zip(p.packets.iter_mut()) {
|
||||
Packet::populate_packet(o, None, i).expect("serialize request");
|
||||
let mut batch = PacketBatch::with_capacity(x.len());
|
||||
batch.packets.resize(x.len(), Packet::default());
|
||||
for (i, packet) in x.iter().zip(batch.packets.iter_mut()) {
|
||||
Packet::populate_packet(packet, None, i).expect("serialize request");
|
||||
}
|
||||
out.push(p);
|
||||
out.push(batch);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn to_packets<T: Serialize>(xs: &[T]) -> Vec<Packets> {
|
||||
to_packets_chunked(xs, NUM_PACKETS)
|
||||
pub fn to_packet_batches_for_tests<T: Serialize>(xs: &[T]) -> Vec<PacketBatch> {
|
||||
to_packet_batches(xs, NUM_PACKETS)
|
||||
}
|
||||
|
||||
pub fn to_packets_with_destination<T: Serialize>(
|
||||
recycler: PacketsRecycler,
|
||||
pub fn to_packet_batch_with_destination<T: Serialize>(
|
||||
recycler: PacketBatchRecycler,
|
||||
dests_and_data: &[(SocketAddr, T)],
|
||||
) -> Packets {
|
||||
let mut out = Packets::new_unpinned_with_recycler(
|
||||
) -> PacketBatch {
|
||||
let mut out = PacketBatch::new_unpinned_with_recycler(
|
||||
recycler,
|
||||
dests_and_data.len(),
|
||||
"to_packets_with_destination",
|
||||
"to_packet_batch_with_destination",
|
||||
);
|
||||
out.packets.resize(dests_and_data.len(), Packet::default());
|
||||
for (dest_and_data, o) in dests_and_data.iter().zip(out.packets.iter_mut()) {
|
||||
@ -143,21 +147,21 @@ mod tests {
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn test_to_packets() {
|
||||
fn test_to_packet_batches() {
|
||||
let keypair = Keypair::new();
|
||||
let hash = Hash::new(&[1; 32]);
|
||||
let tx = system_transaction::transfer(&keypair, &keypair.pubkey(), 1, hash);
|
||||
let rv = to_packets(&[tx.clone(); 1]);
|
||||
let rv = to_packet_batches_for_tests(&[tx.clone(); 1]);
|
||||
assert_eq!(rv.len(), 1);
|
||||
assert_eq!(rv[0].packets.len(), 1);
|
||||
|
||||
#[allow(clippy::useless_vec)]
|
||||
let rv = to_packets(&vec![tx.clone(); NUM_PACKETS]);
|
||||
let rv = to_packet_batches_for_tests(&vec![tx.clone(); NUM_PACKETS]);
|
||||
assert_eq!(rv.len(), 1);
|
||||
assert_eq!(rv[0].packets.len(), NUM_PACKETS);
|
||||
|
||||
#[allow(clippy::useless_vec)]
|
||||
let rv = to_packets(&vec![tx; NUM_PACKETS + 1]);
|
||||
let rv = to_packet_batches_for_tests(&vec![tx; NUM_PACKETS + 1]);
|
||||
assert_eq!(rv.len(), 2);
|
||||
assert_eq!(rv[0].packets.len(), NUM_PACKETS);
|
||||
assert_eq!(rv[1].packets.len(), 1);
|
||||
@ -165,9 +169,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_to_packets_pinning() {
|
||||
let recycler = PacketsRecycler::default();
|
||||
let recycler = PacketBatchRecycler::default();
|
||||
for i in 0..2 {
|
||||
let _first_packets = Packets::new_with_recycler(recycler.clone(), i + 1, "first one");
|
||||
let _first_packets =
|
||||
PacketBatch::new_with_recycler(recycler.clone(), i + 1, "first one");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -182,7 +182,7 @@ impl<T: Default + Reset> RecyclerX<T> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use {super::*, crate::packet::PacketsRecycler, std::iter::repeat_with};
|
||||
use {super::*, crate::packet::PacketBatchRecycler, std::iter::repeat_with};
|
||||
|
||||
impl Reset for u64 {
|
||||
fn reset(&mut self) {
|
||||
@ -209,7 +209,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_recycler_shrink() {
|
||||
let mut rng = rand::thread_rng();
|
||||
let recycler = PacketsRecycler::default();
|
||||
let recycler = PacketBatchRecycler::default();
|
||||
// Allocate a burst of packets.
|
||||
const NUM_PACKETS: usize = RECYCLER_SHRINK_SIZE * 2;
|
||||
{
|
||||
|
||||
@ -9,7 +9,7 @@ use solana_sdk::transaction::Transaction;
|
||||
use {
|
||||
crate::{
|
||||
cuda_runtime::PinnedVec,
|
||||
packet::{Packet, Packets},
|
||||
packet::{Packet, PacketBatch},
|
||||
perf_libs,
|
||||
recycler::Recycler,
|
||||
},
|
||||
@ -158,8 +158,8 @@ fn verify_packet(packet: &mut Packet, reject_non_vote: bool) {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn batch_size(batches: &[Packets]) -> usize {
|
||||
batches.iter().map(|p| p.packets.len()).sum()
|
||||
pub fn count_packets_in_batches(batches: &[PacketBatch]) -> usize {
|
||||
batches.iter().map(|batch| batch.packets.len()).sum()
|
||||
}
|
||||
|
||||
// internal function to be unit-tested; should be used only by get_packet_offsets
|
||||
@ -366,7 +366,7 @@ fn check_for_simple_vote_transaction(
|
||||
}
|
||||
|
||||
pub fn generate_offsets(
|
||||
batches: &mut [Packets],
|
||||
batches: &mut [PacketBatch],
|
||||
recycler: &Recycler<TxOffset>,
|
||||
reject_non_vote: bool,
|
||||
) -> TxOffsets {
|
||||
@ -381,9 +381,9 @@ pub fn generate_offsets(
|
||||
msg_sizes.set_pinnable();
|
||||
let mut current_offset: usize = 0;
|
||||
let mut v_sig_lens = Vec::new();
|
||||
batches.iter_mut().for_each(|p| {
|
||||
batches.iter_mut().for_each(|batch| {
|
||||
let mut sig_lens = Vec::new();
|
||||
p.packets.iter_mut().for_each(|packet| {
|
||||
batch.packets.iter_mut().for_each(|packet| {
|
||||
let packet_offsets = get_packet_offsets(packet, current_offset, reject_non_vote);
|
||||
|
||||
sig_lens.push(packet_offsets.sig_len);
|
||||
@ -418,30 +418,32 @@ pub fn generate_offsets(
|
||||
)
|
||||
}
|
||||
|
||||
pub fn ed25519_verify_cpu(batches: &mut [Packets], reject_non_vote: bool) {
|
||||
pub fn ed25519_verify_cpu(batches: &mut [PacketBatch], reject_non_vote: bool) {
|
||||
use rayon::prelude::*;
|
||||
let count = batch_size(batches);
|
||||
debug!("CPU ECDSA for {}", batch_size(batches));
|
||||
let packet_count = count_packets_in_batches(batches);
|
||||
debug!("CPU ECDSA for {}", packet_count);
|
||||
PAR_THREAD_POOL.install(|| {
|
||||
batches.into_par_iter().for_each(|p| {
|
||||
p.packets
|
||||
batches.into_par_iter().for_each(|batch| {
|
||||
batch
|
||||
.packets
|
||||
.par_iter_mut()
|
||||
.for_each(|p| verify_packet(p, reject_non_vote))
|
||||
})
|
||||
});
|
||||
inc_new_counter_debug!("ed25519_verify_cpu", count);
|
||||
inc_new_counter_debug!("ed25519_verify_cpu", packet_count);
|
||||
}
|
||||
|
||||
pub fn ed25519_verify_disabled(batches: &mut [Packets]) {
|
||||
pub fn ed25519_verify_disabled(batches: &mut [PacketBatch]) {
|
||||
use rayon::prelude::*;
|
||||
let count = batch_size(batches);
|
||||
debug!("disabled ECDSA for {}", batch_size(batches));
|
||||
batches.into_par_iter().for_each(|p| {
|
||||
p.packets
|
||||
let packet_count = count_packets_in_batches(batches);
|
||||
debug!("disabled ECDSA for {}", packet_count);
|
||||
batches.into_par_iter().for_each(|batch| {
|
||||
batch
|
||||
.packets
|
||||
.par_iter_mut()
|
||||
.for_each(|p| p.meta.discard = false)
|
||||
});
|
||||
inc_new_counter_debug!("ed25519_verify_disabled", count);
|
||||
inc_new_counter_debug!("ed25519_verify_disabled", packet_count);
|
||||
}
|
||||
|
||||
pub fn copy_return_values(sig_lens: &[Vec<u32>], out: &PinnedVec<u8>, rvs: &mut Vec<Vec<u8>>) {
|
||||
@ -495,7 +497,7 @@ pub fn get_checked_scalar(scalar: &[u8; 32]) -> Result<[u8; 32], PacketError> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub fn mark_disabled(batches: &mut [Packets], r: &[Vec<u8>]) {
|
||||
pub fn mark_disabled(batches: &mut [PacketBatch], r: &[Vec<u8>]) {
|
||||
batches.iter_mut().zip(r).for_each(|(b, v)| {
|
||||
b.packets.iter_mut().zip(v).for_each(|(p, f)| {
|
||||
p.meta.discard = *f == 0;
|
||||
@ -504,7 +506,7 @@ pub fn mark_disabled(batches: &mut [Packets], r: &[Vec<u8>]) {
|
||||
}
|
||||
|
||||
pub fn ed25519_verify(
|
||||
batches: &mut [Packets],
|
||||
batches: &mut [PacketBatch],
|
||||
recycler: &Recycler<TxOffset>,
|
||||
recycler_out: &Recycler<PinnedVec<u8>>,
|
||||
reject_non_vote: bool,
|
||||
@ -516,21 +518,21 @@ pub fn ed25519_verify(
|
||||
let api = api.unwrap();
|
||||
|
||||
use crate::packet::PACKET_DATA_SIZE;
|
||||
let count = batch_size(batches);
|
||||
let packet_count = count_packets_in_batches(batches);
|
||||
|
||||
// micro-benchmarks show GPU time for smallest batch around 15-20ms
|
||||
// and CPU speed for 64-128 sigverifies around 10-20ms. 64 is a nice
|
||||
// power-of-two number around that accounting for the fact that the CPU
|
||||
// may be busy doing other things while being a real validator
|
||||
// TODO: dynamically adjust this crossover
|
||||
if count < 64 {
|
||||
if packet_count < 64 {
|
||||
return ed25519_verify_cpu(batches, reject_non_vote);
|
||||
}
|
||||
|
||||
let (signature_offsets, pubkey_offsets, msg_start_offsets, msg_sizes, sig_lens) =
|
||||
generate_offsets(batches, recycler, reject_non_vote);
|
||||
|
||||
debug!("CUDA ECDSA for {}", batch_size(batches));
|
||||
debug!("CUDA ECDSA for {}", packet_count);
|
||||
debug!("allocating out..");
|
||||
let mut out = recycler_out.allocate("out_buffer");
|
||||
out.set_pinnable();
|
||||
@ -538,15 +540,15 @@ pub fn ed25519_verify(
|
||||
let mut rvs = Vec::new();
|
||||
|
||||
let mut num_packets: usize = 0;
|
||||
for p in batches.iter() {
|
||||
for batch in batches.iter() {
|
||||
elems.push(perf_libs::Elems {
|
||||
elems: p.packets.as_ptr(),
|
||||
num: p.packets.len() as u32,
|
||||
elems: batch.packets.as_ptr(),
|
||||
num: batch.packets.len() as u32,
|
||||
});
|
||||
let mut v = Vec::new();
|
||||
v.resize(p.packets.len(), 0);
|
||||
v.resize(batch.packets.len(), 0);
|
||||
rvs.push(v);
|
||||
num_packets = num_packets.saturating_add(p.packets.len());
|
||||
num_packets = num_packets.saturating_add(batch.packets.len());
|
||||
}
|
||||
out.resize(signature_offsets.len(), 0);
|
||||
trace!("Starting verify num packets: {}", num_packets);
|
||||
@ -575,7 +577,7 @@ pub fn ed25519_verify(
|
||||
trace!("done verify");
|
||||
copy_return_values(&sig_lens, &out, &mut rvs);
|
||||
mark_disabled(batches, &rvs);
|
||||
inc_new_counter_debug!("ed25519_verify_gpu", count);
|
||||
inc_new_counter_debug!("ed25519_verify_gpu", packet_count);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@ -595,7 +597,7 @@ mod tests {
|
||||
use {
|
||||
super::*,
|
||||
crate::{
|
||||
packet::{Packet, Packets},
|
||||
packet::{Packet, PacketBatch},
|
||||
sigverify::{self, PacketOffsets},
|
||||
test_tx::{test_multisig_tx, test_tx, vote_tx},
|
||||
},
|
||||
@ -623,9 +625,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_mark_disabled() {
|
||||
let mut batch = Packets::default();
|
||||
let mut batch = PacketBatch::default();
|
||||
batch.packets.push(Packet::default());
|
||||
let mut batches: Vec<Packets> = vec![batch];
|
||||
let mut batches: Vec<PacketBatch> = vec![batch];
|
||||
mark_disabled(&mut batches, &[vec![0]]);
|
||||
assert!(batches[0].packets[0].meta.discard);
|
||||
mark_disabled(&mut batches, &[vec![1]]);
|
||||
@ -731,7 +733,7 @@ mod tests {
|
||||
assert!(packet.meta.discard);
|
||||
|
||||
packet.meta.discard = false;
|
||||
let mut batches = generate_packet_vec(&packet, 1, 1);
|
||||
let mut batches = generate_packet_batches(&packet, 1, 1);
|
||||
ed25519_verify(&mut batches);
|
||||
assert!(batches[0].packets[0].meta.discard);
|
||||
}
|
||||
@ -767,7 +769,7 @@ mod tests {
|
||||
assert!(packet.meta.discard);
|
||||
|
||||
packet.meta.discard = false;
|
||||
let mut batches = generate_packet_vec(&packet, 1, 1);
|
||||
let mut batches = generate_packet_batches(&packet, 1, 1);
|
||||
ed25519_verify(&mut batches);
|
||||
assert!(batches[0].packets[0].meta.discard);
|
||||
}
|
||||
@ -929,21 +931,21 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
fn generate_packet_vec(
|
||||
fn generate_packet_batches(
|
||||
packet: &Packet,
|
||||
num_packets_per_batch: usize,
|
||||
num_batches: usize,
|
||||
) -> Vec<Packets> {
|
||||
) -> Vec<PacketBatch> {
|
||||
// generate packet vector
|
||||
let batches: Vec<_> = (0..num_batches)
|
||||
.map(|_| {
|
||||
let mut packets = Packets::default();
|
||||
packets.packets.resize(0, Packet::default());
|
||||
let mut packet_batch = PacketBatch::default();
|
||||
packet_batch.packets.resize(0, Packet::default());
|
||||
for _ in 0..num_packets_per_batch {
|
||||
packets.packets.push(packet.clone());
|
||||
packet_batch.packets.push(packet.clone());
|
||||
}
|
||||
assert_eq!(packets.packets.len(), num_packets_per_batch);
|
||||
packets
|
||||
assert_eq!(packet_batch.packets.len(), num_packets_per_batch);
|
||||
packet_batch
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(batches.len(), num_batches);
|
||||
@ -960,7 +962,7 @@ mod tests {
|
||||
packet.data[20] = packet.data[20].wrapping_add(10);
|
||||
}
|
||||
|
||||
let mut batches = generate_packet_vec(&packet, n, 2);
|
||||
let mut batches = generate_packet_batches(&packet, n, 2);
|
||||
|
||||
// verify packets
|
||||
ed25519_verify(&mut batches);
|
||||
@ -969,11 +971,11 @@ mod tests {
|
||||
let should_discard = modify_data;
|
||||
assert!(batches
|
||||
.iter()
|
||||
.flat_map(|p| &p.packets)
|
||||
.flat_map(|batch| &batch.packets)
|
||||
.all(|p| p.meta.discard == should_discard));
|
||||
}
|
||||
|
||||
fn ed25519_verify(batches: &mut [Packets]) {
|
||||
fn ed25519_verify(batches: &mut [PacketBatch]) {
|
||||
let recycler = Recycler::default();
|
||||
let recycler_out = Recycler::default();
|
||||
sigverify::ed25519_verify(batches, &recycler, &recycler_out, false);
|
||||
@ -986,13 +988,13 @@ mod tests {
|
||||
tx.signatures.pop();
|
||||
let packet = sigverify::make_packet_from_transaction(tx);
|
||||
|
||||
let mut batches = generate_packet_vec(&packet, 1, 1);
|
||||
let mut batches = generate_packet_batches(&packet, 1, 1);
|
||||
|
||||
// verify packets
|
||||
ed25519_verify(&mut batches);
|
||||
assert!(batches
|
||||
.iter()
|
||||
.flat_map(|p| &p.packets)
|
||||
.flat_map(|batch| &batch.packets)
|
||||
.all(|p| p.meta.discard));
|
||||
}
|
||||
|
||||
@ -1020,7 +1022,7 @@ mod tests {
|
||||
|
||||
let n = 4;
|
||||
let num_batches = 3;
|
||||
let mut batches = generate_packet_vec(&packet, n, num_batches);
|
||||
let mut batches = generate_packet_batches(&packet, n, num_batches);
|
||||
|
||||
packet.data[40] = packet.data[40].wrapping_add(8);
|
||||
|
||||
@ -1035,7 +1037,7 @@ mod tests {
|
||||
ref_vec[0].push(0u8);
|
||||
assert!(batches
|
||||
.iter()
|
||||
.flat_map(|p| &p.packets)
|
||||
.flat_map(|batch| &batch.packets)
|
||||
.zip(ref_vec.into_iter().flatten())
|
||||
.all(|(p, discard)| {
|
||||
if discard == 0 {
|
||||
@ -1059,7 +1061,7 @@ mod tests {
|
||||
for _ in 0..50 {
|
||||
let n = thread_rng().gen_range(1, 30);
|
||||
let num_batches = thread_rng().gen_range(2, 30);
|
||||
let mut batches = generate_packet_vec(&packet, n, num_batches);
|
||||
let mut batches = generate_packet_batches(&packet, n, num_batches);
|
||||
|
||||
let num_modifications = thread_rng().gen_range(0, 5);
|
||||
for _ in 0..num_modifications {
|
||||
@ -1080,8 +1082,8 @@ mod tests {
|
||||
// check result
|
||||
batches
|
||||
.iter()
|
||||
.flat_map(|p| &p.packets)
|
||||
.zip(batches_cpu.iter().flat_map(|p| &p.packets))
|
||||
.flat_map(|batch| &batch.packets)
|
||||
.zip(batches_cpu.iter().flat_map(|batch| &batch.packets))
|
||||
.for_each(|(p1, p2)| assert_eq!(p1, p2));
|
||||
}
|
||||
}
|
||||
@ -1233,7 +1235,7 @@ mod tests {
|
||||
solana_logger::setup();
|
||||
|
||||
let mut current_offset = 0usize;
|
||||
let mut batch = Packets::default();
|
||||
let mut batch = PacketBatch::default();
|
||||
batch
|
||||
.packets
|
||||
.push(sigverify::make_packet_from_transaction(test_tx()));
|
||||
|
||||
Reference in New Issue
Block a user