Enable parallel tests to reduce test time (#4919)

* Add crate to serialize some tests

* Ignore unused attribute warning

* Enable parallel run in CI

* Try to fix lograte tests

* Fix interdependent counter tests

Cargo.lock

@@ -2113,6 +2113,23 @@ dependencies = [
  "yaml-rust 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "serial_test"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "serial_test_derive"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "sha-1"
 version = "0.8.1"
@@ -2225,6 +2242,8 @@ dependencies = [
  "serde 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-budget-api 0.17.0",
  "solana-budget-program 0.17.0",
  "solana-chacha-sys 0.17.0",
@@ -2639,6 +2658,8 @@ dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.9.18 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-sdk 0.17.0",
  "sys-info 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -3775,6 +3796,8 @@ dependencies = [
 "checksum serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)" = "5a23aa71d4a4d43fdbfaac00eff68ba8a06a51759a89ac3304323e800c4dd40d"
 "checksum serde_urlencoded 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d48f9f99cd749a2de71d29da5f948de7f2764cc5a9d7f3c97e3514d4ee6eabf2"
 "checksum serde_yaml 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)" = "38b08a9a90e5260fe01c6480ec7c811606df6d3a660415808c3c3fa8ed95b582"
+"checksum serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50bfbc39343545618d97869d77f38ed43e48dd77432717dbc7ed39d797f3ecbe"
+"checksum serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "89dd85be2e2ad75b041c9df2892ac078fa6e0b90024028b2b9fb4125b7530f01"
 "checksum sha-1 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "23962131a91661d643c98940b20fcaffe62d776a823247be80a48fcb8b6fce68"
 "checksum sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9eb6be24e4c23a84d7184280d2722f7f2731fcdd4a9d886efbfe4413e4847ea0"
 "checksum sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b4d8bfd0e469f417657573d8451fb33d16cfe0989359b93baf3a1ffc639543d"

ci/test-stable.sh

@@ -33,7 +33,7 @@ test-stable)
   echo "Executing $testName"
 
   _ cargo +"$rust_stable" build --all ${V:+--verbose}
-  _ cargo +"$rust_stable" test --all ${V:+--verbose} -- --nocapture --test-threads=1
+  _ cargo +"$rust_stable" test --all ${V:+--verbose} -- --nocapture
   ;;
 test-stable-perf)
   echo "Executing $testName"
@@ -77,7 +77,7 @@ test-stable-perf)
 
   # Run root package library tests
   _ cargo +"$rust_stable" build --all ${V:+--verbose} --features="$ROOT_FEATURES"
-  _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1
+  _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture
   ;;
 *)
   echo "Error: Unknown test: $testName"

core/Cargo.toml

@@ -83,6 +83,8 @@ reed-solomon-erasure = "3.1.1"
 [dev-dependencies]
 hex-literal = "0.2.0"
 matches = "0.1.6"
+serial_test = "0.2.0"
+serial_test_derive = "0.2.0"
 
 [[bench]]
 name = "banking_stage"

core/tests/cluster_info.rs

@@ -2,6 +2,7 @@ use rand::SeedableRng;
 use rand_chacha::ChaChaRng;
 use rayon::iter::ParallelIterator;
 use rayon::prelude::*;
+use serial_test_derive::serial;
 use solana::cluster_info::{compute_retransmit_peers, ClusterInfo};
 use solana::contact_info::ContactInfo;
 use solana_sdk::pubkey::Pubkey;
@@ -190,6 +191,7 @@ fn run_simulation(stakes: &[u64], fanout: usize) {
 
 // Run with a single layer
 #[test]
+#[serial]
 fn test_retransmit_small() {
     let stakes: Vec<_> = (0..200).map(|i| i).collect();
     run_simulation(&stakes, 200);
@@ -197,6 +199,7 @@ fn test_retransmit_small() {
 
 // Make sure at least 2 layers are used
 #[test]
+#[serial]
 fn test_retransmit_medium() {
     let num_nodes = 2000;
     let stakes: Vec<_> = (0..num_nodes).map(|i| i).collect();
@@ -205,6 +208,7 @@ fn test_retransmit_medium() {
 
 // Make sure at least 2 layers are used but with equal stakes
 #[test]
+#[serial]
 fn test_retransmit_medium_equal_stakes() {
     let num_nodes = 2000;
     let stakes: Vec<_> = (0..num_nodes).map(|_| 10).collect();
@@ -213,6 +217,7 @@ fn test_retransmit_medium_equal_stakes() {
 
 // Scale down the network and make sure many layers are used
 #[test]
+#[serial]
 fn test_retransmit_large() {
     let num_nodes = 4000;
     let stakes: Vec<_> = (0..num_nodes).map(|i| i).collect();

core/tests/local_cluster.rs

@@ -2,6 +2,7 @@ extern crate solana;
 
 use hashbrown::HashSet;
 use log::*;
+use serial_test_derive::serial;
 use solana::broadcast_stage::BroadcastStageType;
 use solana::cluster::Cluster;
 use solana::cluster_tests;
@@ -16,6 +17,7 @@ use std::thread::sleep;
 use std::time::Duration;
 
 #[test]
+#[serial]
 fn test_spend_and_verify_all_nodes_1() {
     solana_logger::setup();
     let num_nodes = 1;
@@ -29,6 +31,7 @@ fn test_spend_and_verify_all_nodes_1() {
 }
 
 #[test]
+#[serial]
 fn test_spend_and_verify_all_nodes_2() {
     solana_logger::setup();
     let num_nodes = 2;
@@ -42,6 +45,7 @@ fn test_spend_and_verify_all_nodes_2() {
 }
 
 #[test]
+#[serial]
 fn test_spend_and_verify_all_nodes_3() {
     solana_logger::setup();
     let num_nodes = 3;
@@ -54,7 +58,9 @@ fn test_spend_and_verify_all_nodes_3() {
     );
 }
 
+#[allow(unused_attributes)]
 #[test]
+#[serial]
 #[ignore]
 fn test_spend_and_verify_all_nodes_env_num_nodes() {
     solana_logger::setup();
@@ -71,7 +77,9 @@ fn test_spend_and_verify_all_nodes_env_num_nodes() {
     );
 }
 
+#[allow(unused_attributes)]
 #[test]
+#[serial]
 #[should_panic]
 fn test_fullnode_exit_default_config_should_panic() {
     solana_logger::setup();
@@ -81,6 +89,7 @@ fn test_fullnode_exit_default_config_should_panic() {
 }
 
 #[test]
+#[serial]
 fn test_fullnode_exit_2() {
     solana_logger::setup();
     let num_nodes = 2;
@@ -98,6 +107,7 @@ fn test_fullnode_exit_2() {
 
 // Cluster needs a supermajority to remain, so the minimum size for this test is 4
 #[test]
+#[serial]
 fn test_leader_failure_4() {
     solana_logger::setup();
     let num_nodes = 4;
@@ -118,6 +128,7 @@ fn test_leader_failure_4() {
     );
 }
 #[test]
+#[serial]
 fn test_two_unbalanced_stakes() {
     solana_logger::setup();
     let mut validator_config = ValidatorConfig::default();
@@ -179,6 +190,7 @@ fn test_forwarding() {
 }
 
 #[test]
+#[serial]
 fn test_restart_node() {
     let slots_per_epoch = MINIMUM_SLOTS_PER_EPOCH as u64;
     let ticks_per_slot = 16;
@@ -208,6 +220,7 @@ fn test_restart_node() {
 }
 
 #[test]
+#[serial]
 fn test_listener_startup() {
     let config = ClusterConfig {
         node_stakes: vec![100; 1],
@@ -222,6 +235,7 @@ fn test_listener_startup() {
 }
 
 #[test]
+#[serial]
 fn test_fail_entry_verification_leader() {
     solana_logger::setup();
     let num_nodes = 4;
@@ -329,6 +343,7 @@ fn test_fake_blobs_broadcast_leader() {
 }
 
 #[test]
+#[serial]
 fn test_repairman_catchup() {
     run_repairman_catchup(3);
 }

core/tests/replicator.rs

@@ -4,6 +4,7 @@ extern crate log;
 #[macro_use]
 extern crate solana;
 
+use serial_test_derive::serial;
 use solana::blocktree::{create_new_tmp_ledger, get_tmp_ledger_path, Blocktree};
 use solana::cluster_info::{ClusterInfo, Node, FULLNODE_PORT_RANGE};
 use solana::contact_info::ContactInfo;
@@ -67,16 +68,19 @@ fn run_replicator_startup_basic(num_nodes: usize, num_replicators: usize) {
 }
 
 #[test]
+#[serial]
 fn test_replicator_startup_1_node() {
     run_replicator_startup_basic(1, 1);
 }
 
 #[test]
+#[serial]
 fn test_replicator_startup_2_nodes() {
     run_replicator_startup_basic(2, 1);
 }
 
 #[test]
+#[serial]
 fn test_replicator_startup_leader_hang() {
     use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 
@@ -115,6 +119,7 @@ fn test_replicator_startup_leader_hang() {
 }
 
 #[test]
+#[serial]
 fn test_replicator_startup_ledger_hang() {
     solana_logger::setup();
     info!("starting replicator test");
@@ -143,6 +148,7 @@ fn test_replicator_startup_ledger_hang() {
 }
 
 #[test]
+#[serial]
 fn test_account_setup() {
     let num_nodes = 1;
     let num_replicators = 1;

metrics/Cargo.toml

@@ -19,6 +19,8 @@ sys-info = "0.5.7"
 
 [dev-dependencies]
 rand = "0.6.5"
+serial_test = "0.2.0"
+serial_test_derive = "0.2.0"
 
 [lib]
 name = "solana_metrics"

metrics/src/counter.rs

@@ -194,6 +194,7 @@ mod tests {
     use crate::counter::{Counter, DEFAULT_LOG_RATE};
     use log::Level;
     use log::*;
+    use serial_test_derive::serial;
     use std::env;
     use std::sync::atomic::Ordering;
     use std::sync::{Once, RwLock, ONCE_INIT};
@@ -211,9 +212,11 @@ mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_counter() {
         env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info"))
-            .init();
+            .try_init()
+            .ok();
         let _readlock = get_env_lock().read();
         static mut COUNTER: Counter = create_counter!("test", 1000, 1);
         let count = 1;
@@ -237,6 +240,7 @@ mod tests {
         }
     }
     #[test]
+    #[serial]
     fn test_inc_new_counter() {
         let _readlock = get_env_lock().read();
         //make sure that macros are syntactically correct
@@ -246,7 +250,11 @@ mod tests {
         inc_new_counter_info!("3", 1, 2, 1);
     }
     #[test]
+    #[serial]
     fn test_lograte() {
+        env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info"))
+            .try_init()
+            .ok();
         let _readlock = get_env_lock().read();
         assert_eq!(
             Counter::default_log_rate(),
@@ -263,7 +271,11 @@ mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_lograte_env() {
+        env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info"))
+            .try_init()
+            .ok();
         assert_ne!(DEFAULT_LOG_RATE, 0);
         let _writelock = get_env_lock().write();
         static mut COUNTER: Counter = create_counter!("test_lograte_env", 0, 1);

scripts/coverage.sh

@@ -36,7 +36,7 @@ rm -rf target/cov/$reportName
 
 source ci/rust-version.sh nightly
 _ cargo +$rust_nightly build --target-dir target/cov "$crate"
-_ cargo +$rust_nightly test --target-dir target/cov --lib "$crate" -- --test-threads=1
+_ cargo +$rust_nightly test --target-dir target/cov --lib "$crate"
 
 echo "--- grcov"