Refactor packing packets into blobs into separate packets_to_blob() function in packets.rs

core/src/packet.rs

@@ -1,12 +1,13 @@
 //! The `packet` module defines data structures and methods to pull data from the network.
 use crate::recvmmsg::{recv_mmsg, NUM_RCVMMSGS};
 use crate::result::{Error, Result};
-use bincode::{serialize, serialize_into};
+use bincode;
 use byteorder::{ByteOrder, LittleEndian};
 use serde::Serialize;
 use solana_metrics::counter::Counter;
 pub use solana_sdk::packet::PACKET_DATA_SIZE;
 use solana_sdk::pubkey::Pubkey;
+use std::borrow::Borrow;
 use std::cmp;
 use std::fmt;
 use std::io;
@@ -243,7 +244,7 @@ pub fn to_packets_chunked<T: Serialize>(xs: &[T], chunks: usize) -> Vec<SharedPa
             .resize(x.len(), Packet::default());
         for (i, o) in x.iter().zip(p.write().unwrap().packets.iter_mut()) {
             let mut wr = io::Cursor::new(&mut o.data[..]);
-            serialize_into(&mut wr, &i).expect("serialize request");
+            bincode::serialize_into(&mut wr, &i).expect("serialize request");
             let len = wr.position() as usize;
             o.meta.size = len;
         }
@@ -258,7 +259,7 @@ pub fn to_packets<T: Serialize>(xs: &[T]) -> Vec<SharedPackets> {
 
 pub fn to_blob<T: Serialize>(resp: T, rsp_addr: SocketAddr) -> Result<Blob> {
     let mut b = Blob::default();
-    let v = serialize(&resp)?;
+    let v = bincode::serialize(&resp)?;
     let len = v.len();
     assert!(len <= BLOB_SIZE);
     b.data[..len].copy_from_slice(&v);
@@ -288,6 +289,44 @@ pub fn to_shared_blobs<T: Serialize>(rsps: Vec<(T, SocketAddr)>) -> Result<Share
     Ok(blobs)
 }
 
+pub fn packets_to_blobs<T: Borrow<Packet>>(packets: &[T]) -> Vec<Blob> {
+    let mut current_index = 0;
+    let mut blobs = vec![];
+    while current_index < packets.len() {
+        let mut blob = Blob::default();
+        current_index += blob.store_packets(&packets[current_index..]) as usize;
+        blobs.push(blob);
+    }
+
+    blobs
+}
+
+pub fn deserialize_packets_in_blob(
+    data: &[u8],
+    serialized_packet_size: usize,
+    serialized_meta_size: usize,
+) -> Result<Vec<Packet>> {
+    let mut packets: Vec<Packet> = Vec::with_capacity(data.len() / serialized_packet_size);
+    let mut pos = 0;
+    while pos + serialized_packet_size <= data.len() {
+        let packet = deserialize_single_packet_in_blob(
+            &data[pos..pos + serialized_packet_size],
+            serialized_meta_size,
+        )?;
+        pos += serialized_packet_size;
+        packets.push(packet);
+    }
+    Ok(packets)
+}
+
+fn deserialize_single_packet_in_blob(data: &[u8], serialized_meta_size: usize) -> Result<Packet> {
+    let meta = bincode::deserialize(&data[..serialized_meta_size])?;
+    let mut packet_data = [0; PACKET_DATA_SIZE];
+    packet_data
+        .copy_from_slice(&data[serialized_meta_size..serialized_meta_size + PACKET_DATA_SIZE]);
+    Ok(Packet::new(packet_data, meta))
+}
+
 macro_rules! range {
     ($prev:expr, $type:ident) => {
         $prev..$prev + size_of::<$type>()
@@ -419,16 +458,16 @@ impl Blob {
         self.set_data_size(new_size as u64);
     }
 
-    pub fn store_packets(&mut self, packets: &[Packet]) -> u64 {
+    pub fn store_packets<T: Borrow<Packet>>(&mut self, packets: &[T]) -> u64 {
         let size = self.size();
         let mut cursor = Cursor::new(&mut self.data_mut()[size..]);
         let mut written = 0;
         let mut last_index = 0;
         for packet in packets {
-            if serialize_into(&mut cursor, &packet.meta).is_err() {
+            if bincode::serialize_into(&mut cursor, &packet.borrow().meta).is_err() {
                 break;
             }
-            if cursor.write_all(&packet.data[..]).is_err() {
+            if cursor.write_all(&packet.borrow().data[..]).is_err() {
                 break;
             }
 
@@ -518,12 +557,9 @@ pub fn index_blobs(blobs: &[SharedBlob], id: &Pubkey, mut blob_index: u64, slot:
 
 #[cfg(test)]
 mod tests {
-    use crate::packet::{
-        to_packets, Blob, Meta, Packet, Packets, SharedBlob, SharedPackets, NUM_PACKETS,
-        PACKET_DATA_SIZE,
-    };
-    use crate::packet::{BLOB_HEADER_SIZE, BLOB_SIZE};
-    use bincode::serialized_size;
+    use super::*;
+    use bincode;
+    use rand::Rng;
     use solana_sdk::hash::Hash;
     use solana_sdk::signature::{Keypair, KeypairUtil};
     use solana_sdk::system_transaction::SystemTransaction;
@@ -642,7 +678,7 @@ mod tests {
     #[test]
     fn test_store_blobs_max() {
         let meta = Meta::default();
-        let serialized_meta_size = serialized_size(&meta).unwrap() as usize;
+        let serialized_meta_size = bincode::serialized_size(&meta).unwrap() as usize;
         let serialized_packet_size = serialized_meta_size + PACKET_DATA_SIZE;
         let num_packets = (BLOB_SIZE - BLOB_HEADER_SIZE) / serialized_packet_size + 1;
         let mut blob = Blob::default();
@@ -664,4 +700,78 @@ mod tests {
         // Blob is now full
         assert_eq!(blob.store_packets(&packets), 0);
     }
+
+    #[test]
+    fn test_packets_to_blobs() {
+        let mut rng = rand::thread_rng();
+        let meta = Meta::default();
+        let serialized_meta_size = bincode::serialized_size(&meta).unwrap() as usize;
+        let serialized_packet_size = serialized_meta_size + PACKET_DATA_SIZE;
+
+        let packets_per_blob = (BLOB_SIZE - BLOB_HEADER_SIZE) / serialized_packet_size;
+        assert!(packets_per_blob > 1);
+        let num_packets = packets_per_blob * 10 + packets_per_blob - 1;
+
+        let packets: Vec<_> = (0..num_packets)
+            .map(|_| {
+                let mut packet = Packet::default();
+                for i in 0..packet.meta.addr.len() {
+                    packet.meta.addr[i] = rng.gen_range(1, std::u16::MAX);
+                }
+                for i in 0..packet.data.len() {
+                    packet.data[i] = rng.gen_range(1, std::u8::MAX);
+                }
+                packet
+            })
+            .collect();
+
+        let blobs = packets_to_blobs(&packets[..]);
+        assert_eq!(blobs.len(), 11);
+
+        let reconstructed_packets: Vec<Packet> = blobs
+            .iter()
+            .flat_map(|b| {
+                deserialize_packets_in_blob(
+                    &b.data()[..b.size()],
+                    serialized_packet_size,
+                    serialized_meta_size,
+                )
+                .unwrap()
+            })
+            .collect();
+
+        assert_eq!(reconstructed_packets, packets);
+    }
+
+    #[test]
+    fn test_deserialize_packets_in_blob() {
+        let meta = Meta::default();
+        let serialized_meta_size = bincode::serialized_size(&meta).unwrap() as usize;
+        let serialized_packet_size = serialized_meta_size + PACKET_DATA_SIZE;
+        let num_packets = 10;
+        let mut rng = rand::thread_rng();
+        let packets: Vec<_> = (0..num_packets)
+            .map(|_| {
+                let mut packet = Packet::default();
+                for i in 0..packet.meta.addr.len() {
+                    packet.meta.addr[i] = rng.gen_range(1, std::u16::MAX);
+                }
+                for i in 0..packet.data.len() {
+                    packet.data[i] = rng.gen_range(1, std::u8::MAX);
+                }
+                packet
+            })
+            .collect();
+
+        let mut blob = Blob::default();
+        assert_eq!(blob.store_packets(&packets[..]), num_packets);
+        let result = deserialize_packets_in_blob(
+            &blob.data()[..blob.size()],
+            serialized_packet_size,
+            serialized_meta_size,
+        )
+        .unwrap();
+
+        assert_eq!(result, packets);
+    }
 }