Enable parallel tests to reduce test time (#4919)

* Add crate to serialize some tests

* Ignore unused attribute warning

* Enable parallel run in CI

* Try to fix lograte tests

* Fix interdependent counter tests

core/Cargo.toml

@@ -83,6 +83,8 @@ reed-solomon-erasure = "3.1.1"
 [dev-dependencies]
 hex-literal = "0.2.0"
 matches = "0.1.6"
+serial_test = "0.2.0"
+serial_test_derive = "0.2.0"
 
 [[bench]]
 name = "banking_stage"

core/tests/cluster_info.rs

@@ -2,6 +2,7 @@ use rand::SeedableRng;
 use rand_chacha::ChaChaRng;
 use rayon::iter::ParallelIterator;
 use rayon::prelude::*;
+use serial_test_derive::serial;
 use solana::cluster_info::{compute_retransmit_peers, ClusterInfo};
 use solana::contact_info::ContactInfo;
 use solana_sdk::pubkey::Pubkey;
@@ -190,6 +191,7 @@ fn run_simulation(stakes: &[u64], fanout: usize) {
 
 // Run with a single layer
 #[test]
+#[serial]
 fn test_retransmit_small() {
     let stakes: Vec<_> = (0..200).map(|i| i).collect();
     run_simulation(&stakes, 200);
@@ -197,6 +199,7 @@ fn test_retransmit_small() {
 
 // Make sure at least 2 layers are used
 #[test]
+#[serial]
 fn test_retransmit_medium() {
     let num_nodes = 2000;
     let stakes: Vec<_> = (0..num_nodes).map(|i| i).collect();
@@ -205,6 +208,7 @@ fn test_retransmit_medium() {
 
 // Make sure at least 2 layers are used but with equal stakes
 #[test]
+#[serial]
 fn test_retransmit_medium_equal_stakes() {
     let num_nodes = 2000;
     let stakes: Vec<_> = (0..num_nodes).map(|_| 10).collect();
@@ -213,6 +217,7 @@ fn test_retransmit_medium_equal_stakes() {
 
 // Scale down the network and make sure many layers are used
 #[test]
+#[serial]
 fn test_retransmit_large() {
     let num_nodes = 4000;
     let stakes: Vec<_> = (0..num_nodes).map(|i| i).collect();

core/tests/local_cluster.rs

@@ -2,6 +2,7 @@ extern crate solana;
 
 use hashbrown::HashSet;
 use log::*;
+use serial_test_derive::serial;
 use solana::broadcast_stage::BroadcastStageType;
 use solana::cluster::Cluster;
 use solana::cluster_tests;
@@ -16,6 +17,7 @@ use std::thread::sleep;
 use std::time::Duration;
 
 #[test]
+#[serial]
 fn test_spend_and_verify_all_nodes_1() {
     solana_logger::setup();
     let num_nodes = 1;
@@ -29,6 +31,7 @@ fn test_spend_and_verify_all_nodes_1() {
 }
 
 #[test]
+#[serial]
 fn test_spend_and_verify_all_nodes_2() {
     solana_logger::setup();
     let num_nodes = 2;
@@ -42,6 +45,7 @@ fn test_spend_and_verify_all_nodes_2() {
 }
 
 #[test]
+#[serial]
 fn test_spend_and_verify_all_nodes_3() {
     solana_logger::setup();
     let num_nodes = 3;
@@ -54,7 +58,9 @@ fn test_spend_and_verify_all_nodes_3() {
     );
 }
 
+#[allow(unused_attributes)]
 #[test]
+#[serial]
 #[ignore]
 fn test_spend_and_verify_all_nodes_env_num_nodes() {
     solana_logger::setup();
@@ -71,7 +77,9 @@ fn test_spend_and_verify_all_nodes_env_num_nodes() {
     );
 }
 
+#[allow(unused_attributes)]
 #[test]
+#[serial]
 #[should_panic]
 fn test_fullnode_exit_default_config_should_panic() {
     solana_logger::setup();
@@ -81,6 +89,7 @@ fn test_fullnode_exit_default_config_should_panic() {
 }
 
 #[test]
+#[serial]
 fn test_fullnode_exit_2() {
     solana_logger::setup();
     let num_nodes = 2;
@@ -98,6 +107,7 @@ fn test_fullnode_exit_2() {
 
 // Cluster needs a supermajority to remain, so the minimum size for this test is 4
 #[test]
+#[serial]
 fn test_leader_failure_4() {
     solana_logger::setup();
     let num_nodes = 4;
@@ -118,6 +128,7 @@ fn test_leader_failure_4() {
     );
 }
 #[test]
+#[serial]
 fn test_two_unbalanced_stakes() {
     solana_logger::setup();
     let mut validator_config = ValidatorConfig::default();
@@ -179,6 +190,7 @@ fn test_forwarding() {
 }
 
 #[test]
+#[serial]
 fn test_restart_node() {
     let slots_per_epoch = MINIMUM_SLOTS_PER_EPOCH as u64;
     let ticks_per_slot = 16;
@@ -208,6 +220,7 @@ fn test_restart_node() {
 }
 
 #[test]
+#[serial]
 fn test_listener_startup() {
     let config = ClusterConfig {
         node_stakes: vec![100; 1],
@@ -222,6 +235,7 @@ fn test_listener_startup() {
 }
 
 #[test]
+#[serial]
 fn test_fail_entry_verification_leader() {
     solana_logger::setup();
     let num_nodes = 4;
@@ -329,6 +343,7 @@ fn test_fake_blobs_broadcast_leader() {
 }
 
 #[test]
+#[serial]
 fn test_repairman_catchup() {
     run_repairman_catchup(3);
 }

core/tests/replicator.rs

@@ -4,6 +4,7 @@ extern crate log;
 #[macro_use]
 extern crate solana;
 
+use serial_test_derive::serial;
 use solana::blocktree::{create_new_tmp_ledger, get_tmp_ledger_path, Blocktree};
 use solana::cluster_info::{ClusterInfo, Node, FULLNODE_PORT_RANGE};
 use solana::contact_info::ContactInfo;
@@ -67,16 +68,19 @@ fn run_replicator_startup_basic(num_nodes: usize, num_replicators: usize) {
 }
 
 #[test]
+#[serial]
 fn test_replicator_startup_1_node() {
     run_replicator_startup_basic(1, 1);
 }
 
 #[test]
+#[serial]
 fn test_replicator_startup_2_nodes() {
     run_replicator_startup_basic(2, 1);
 }
 
 #[test]
+#[serial]
 fn test_replicator_startup_leader_hang() {
     use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 
@@ -115,6 +119,7 @@ fn test_replicator_startup_leader_hang() {
 }
 
 #[test]
+#[serial]
 fn test_replicator_startup_ledger_hang() {
     solana_logger::setup();
     info!("starting replicator test");
@@ -143,6 +148,7 @@ fn test_replicator_startup_ledger_hang() {
 }
 
 #[test]
+#[serial]
 fn test_account_setup() {
     let num_nodes = 1;
     let num_replicators = 1;