Add snapshotting integration test (#5519)

* Add snapshotting integration test

* Update ContactInfo on restart in local cluster nodes

local_cluster/src/cluster_tests.rs

@@ -1,3 +1,4 @@
+use rand::{thread_rng, Rng};
 use solana_client::thin_client::create_client;
 /// Cluster independant integration tests
 ///
@@ -25,7 +26,12 @@ use solana_sdk::{
     },
     transport::TransportError,
 };
-use std::{collections::HashSet, path::Path, thread::sleep, time::Duration};
+use std::{
+    collections::{HashMap, HashSet},
+    path::Path,
+    thread::sleep,
+    time::Duration,
+};
 
 const DEFAULT_SLOT_MILLIS: u64 = (DEFAULT_TICKS_PER_SLOT * 1000) / DEFAULT_TICKS_PER_SECOND;
 
@@ -65,8 +71,25 @@ pub fn spend_and_verify_all_nodes<S: ::std::hash::BuildHasher>(
     }
 }
 
-pub fn send_many_transactions(node: &ContactInfo, funding_keypair: &Keypair, num_txs: u64) {
+pub fn verify_balances<S: ::std::hash::BuildHasher>(
+    expected_balances: HashMap<Pubkey, u64, S>,
+    node: &ContactInfo,
+) {
     let client = create_client(node.client_facing_addr(), FULLNODE_PORT_RANGE);
+    for (pk, b) in expected_balances {
+        let bal = client.poll_get_balance(&pk).expect("balance in source");
+        assert_eq!(bal, b);
+    }
+}
+
+pub fn send_many_transactions(
+    node: &ContactInfo,
+    funding_keypair: &Keypair,
+    max_tokens_per_transfer: u64,
+    num_txs: u64,
+) -> HashMap<Pubkey, u64> {
+    let client = create_client(node.client_facing_addr(), FULLNODE_PORT_RANGE);
+    let mut expected_balances = HashMap::new();
     for _ in 0..num_txs {
         let random_keypair = Keypair::new();
         let bal = client
@@ -74,12 +97,23 @@ pub fn send_many_transactions(node: &ContactInfo, funding_keypair: &Keypair, num
             .expect("balance in source");
         assert!(bal > 0);
         let (blockhash, _fee_calculator) = client.get_recent_blockhash().unwrap();
-        let mut transaction =
-            system_transaction::transfer(&funding_keypair, &random_keypair.pubkey(), 1, blockhash);
+        let transfer_amount = thread_rng().gen_range(1, max_tokens_per_transfer);
+
+        let mut transaction = system_transaction::transfer(
+            &funding_keypair,
+            &random_keypair.pubkey(),
+            transfer_amount,
+            blockhash,
+        );
+
         client
             .retry_transfer(&funding_keypair, &mut transaction, 5)
             .unwrap();
+
+        expected_balances.insert(random_keypair.pubkey(), transfer_amount);
     }
+
+    expected_balances
 }
 
 pub fn fullnode_exit(entry_point_info: &ContactInfo, nodes: usize) {

local_cluster/src/lib.rs

@@ -9,3 +9,5 @@ extern crate solana_core;
 
 #[macro_use]
 extern crate solana_storage_program;
+
+extern crate tempfile;

local_cluster/src/local_cluster.rs

@@ -585,19 +585,28 @@ impl Cluster for LocalCluster {
         })
     }
 
-    fn restart_node(&mut self, pubkey: Pubkey) {
+    fn restart_node(&mut self, pubkey: Pubkey, config: &ValidatorConfig) {
         // Shut down the fullnode
         let mut node = self.fullnodes.remove(&pubkey).unwrap();
         node.exit();
         node.join().unwrap();
 
-        // Restart the node
-        let fullnode_info = &self.fullnode_infos[&pubkey].info;
-        let config = &self.fullnode_infos[&pubkey].config;
-        let node = Node::new_localhost_with_pubkey(&fullnode_info.keypair.pubkey());
+        // Update the stored ContactInfo for this node
+        let node_pubkey = &self.fullnode_infos[&pubkey].info.keypair.pubkey();
+        let node = Node::new_localhost_with_pubkey(&node_pubkey);
+        self.fullnode_infos
+            .get_mut(&pubkey)
+            .unwrap()
+            .info
+            .contact_info = node.info.clone();
         if pubkey == self.entry_point_info.id {
             self.entry_point_info = node.info.clone();
         }
+
+        // Restart the node
+        self.fullnode_infos.get_mut(&pubkey).unwrap().config = config.clone();
+        let fullnode_info = &self.fullnode_infos[&pubkey].info;
+
         let restarted_node = Validator::new(
             node,
             &fullnode_info.keypair,