Performance tweaks (#4340)

* Use Rc to prevent clone of Packets

* Fix min => max in banking_stage threads.

Coalesce packet buffers better since a larger batch will
be faster through banking and sigverify.

Deconstruct batches into banking_stage from sigverify since
sigverify likes to accumulate batches but then a single banking_stage
thread will be stuck with a large batch. Maximize parallelism by
creating more chunks of work for banking_stage.

core/src/packet.rs

@@ -18,6 +18,7 @@ use std::mem::size_of;
 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, UdpSocket};
 use std::ops::{Deref, DerefMut};
 use std::sync::{Arc, RwLock};
+use std::time::Instant;
 
 pub type SharedBlob = Arc<RwLock<Blob>>;
 pub type SharedBlobs = Vec<SharedBlob>;
@@ -213,6 +214,9 @@ pub enum BlobError {
 
 impl Packets {
     pub fn recv_from(&mut self, socket: &UdpSocket) -> Result<usize> {
+        const MAX_PACKETS_PER_VEC: usize = 1024;
+        const MAX_MILLIS_PER_BATCH: u128 = 2;
+
         let mut i = 0;
         //DOCUMENTED SIDE-EFFECT
         //Performance out of the IO without poll
@@ -222,11 +226,16 @@ impl Packets {
         //  * set it back to blocking before returning
         socket.set_nonblocking(false)?;
         trace!("receiving on {}", socket.local_addr().unwrap());
+        let start = Instant::now();
         loop {
             self.packets.resize(i + NUM_RCVMMSGS, Packet::default());
             match recv_mmsg(socket, &mut self.packets[i..]) {
                 Err(_) if i > 0 => {
-                    break;
+                    if i >= MAX_PACKETS_PER_VEC
+                        || start.elapsed().as_millis() > MAX_MILLIS_PER_BATCH
+                    {
+                        break;
+                    }
                 }
                 Err(e) => {
                     trace!("recv_from err {:?}", e);
@@ -238,7 +247,9 @@ impl Packets {
                     }
                     trace!("got {} packets", npkts);
                     i += npkts;
-                    if npkts != NUM_RCVMMSGS || i >= 1024 {
+                    if i >= MAX_PACKETS_PER_VEC
+                        || start.elapsed().as_millis() > MAX_MILLIS_PER_BATCH
+                    {
                         break;
                     }
                 }