Fix TVU and PoH Recorder going out of sync (#3164)

* Fix broadcast_stage error

* Account for very fast ticks in tick verification

core/src/local_cluster.rs

@@ -21,33 +21,26 @@ pub struct LocalCluster {
     pub funding_keypair: Keypair,
     /// Entry point from which the rest of the network can be discovered
     pub entry_point_info: NodeInfo,
+    pub ledger_paths: Vec<String>,
     fullnodes: Vec<Fullnode>,
-    ledger_paths: Vec<String>,
 }
 
 impl LocalCluster {
     pub fn new(num_nodes: usize, cluster_lamports: u64, lamports_per_node: u64) -> Self {
-        Self::new_with_config(
-            num_nodes,
-            cluster_lamports,
-            lamports_per_node,
-            &FullnodeConfig::default(),
-        )
+        let stakes: Vec<_> = (0..num_nodes).map(|_| lamports_per_node).collect();
+        Self::new_with_config(&stakes, cluster_lamports, &FullnodeConfig::default())
     }
 
     pub fn new_with_config(
-        num_nodes: usize,
+        node_stakes: &[u64],
         cluster_lamports: u64,
-        lamports_per_node: u64,
         fullnode_config: &FullnodeConfig,
     ) -> Self {
-        // Must have enough tokens to fund vote account and set delegate
-        assert!(lamports_per_node > 2);
         let leader_keypair = Arc::new(Keypair::new());
         let leader_pubkey = leader_keypair.pubkey();
         let leader_node = Node::new_localhost_with_pubkey(leader_keypair.pubkey());
         let (genesis_block, mint_keypair) =
-            GenesisBlock::new_with_leader(cluster_lamports, leader_pubkey, lamports_per_node);
+            GenesisBlock::new_with_leader(cluster_lamports, leader_pubkey, node_stakes[0]);
         let (genesis_ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_block);
         let leader_ledger_path = tmp_copy_blocktree!(&genesis_ledger_path);
         let mut ledger_paths = vec![];
@@ -65,7 +58,9 @@ impl LocalCluster {
         );
         let mut fullnodes = vec![leader_server];
         let mut client = mk_client(&leader_node_info);
-        for _ in 0..(num_nodes - 1) {
+        for stake in &node_stakes[1..] {
+            // Must have enough tokens to fund vote account and set delegate
+            assert!(*stake > 2);
             let validator_keypair = Arc::new(Keypair::new());
             let voting_keypair = Keypair::new();
             let validator_pubkey = validator_keypair.pubkey();
@@ -74,12 +69,8 @@ impl LocalCluster {
             ledger_paths.push(ledger_path.clone());
 
             // Send each validator some lamports to vote
-            let validator_balance = Self::transfer(
-                &mut client,
-                &mint_keypair,
-                &validator_pubkey,
-                lamports_per_node,
-            );
+            let validator_balance =
+                Self::transfer(&mut client, &mint_keypair, &validator_pubkey, *stake);
             info!(
                 "validator {} balance {}",
                 validator_pubkey, validator_balance
@@ -89,7 +80,7 @@ impl LocalCluster {
                 &mut client,
                 &voting_keypair,
                 &validator_keypair,
-                lamports_per_node - 1,
+                stake - 1,
             )
             .unwrap();
             let validator_server = Fullnode::new(
@@ -102,7 +93,7 @@ impl LocalCluster {
             );
             fullnodes.push(validator_server);
         }
-        discover(&leader_node_info, num_nodes).unwrap();
+        discover(&leader_node_info, node_stakes.len()).unwrap();
         Self {
             funding_keypair: mint_keypair,
             entry_point_info: leader_node_info,
@@ -116,11 +107,16 @@ impl LocalCluster {
             node.exit();
         }
     }
-    pub fn close(&mut self) {
+
+    pub fn close_preserve_ledgers(&mut self) {
         self.exit();
         while let Some(node) = self.fullnodes.pop() {
             node.join().unwrap();
         }
+    }
+
+    pub fn close(&mut self) {
+        self.close_preserve_ledgers();
         for path in &self.ledger_paths {
             remove_dir_all(path).unwrap();
         }
@@ -204,7 +200,7 @@ impl LocalCluster {
 
 impl Drop for LocalCluster {
     fn drop(&mut self) {
-        self.close()
+        self.close();
     }
 }
 
@@ -224,7 +220,7 @@ mod test {
         solana_logger::setup();
         let mut fullnode_exit = FullnodeConfig::default();
         fullnode_exit.rpc_config.enable_fullnode_exit = true;
-        let cluster = LocalCluster::new_with_config(1, 100, 3, &fullnode_exit);
+        let cluster = LocalCluster::new_with_config(&[3], 100, &fullnode_exit);
         drop(cluster)
     }
 }