Add test for making sure switch doesn't happen past failure threshold (#11138)

Fix switch threshold

Co-authored-by: Carl <carl@solana.com>

local-cluster/src/cluster_tests.rs

@@ -304,6 +304,60 @@ pub fn check_for_new_roots(num_new_roots: usize, contact_infos: &[ContactInfo],
     }
 }
 
+pub fn check_no_new_roots(
+    num_slots_to_wait: usize,
+    contact_infos: &[ContactInfo],
+    test_name: &str,
+) {
+    assert!(!contact_infos.is_empty());
+    let mut roots = vec![0; contact_infos.len()];
+    let max_slot = contact_infos
+        .iter()
+        .enumerate()
+        .map(|(i, ingress_node)| {
+            let client = create_client(ingress_node.client_facing_addr(), VALIDATOR_PORT_RANGE);
+            let initial_root = client
+                .get_slot()
+                .unwrap_or_else(|_| panic!("get_slot for {} failed", ingress_node.id));
+            roots[i] = initial_root;
+            client
+                .get_slot_with_commitment(CommitmentConfig::recent())
+                .unwrap_or_else(|_| panic!("get_slot for {} failed", ingress_node.id))
+        })
+        .max()
+        .unwrap();
+
+    let end_slot = max_slot + num_slots_to_wait as u64;
+    let mut current_slot;
+    let mut last_print = Instant::now();
+    let client = create_client(contact_infos[0].client_facing_addr(), VALIDATOR_PORT_RANGE);
+    loop {
+        current_slot = client
+            .get_slot_with_commitment(CommitmentConfig::recent())
+            .unwrap_or_else(|_| panic!("get_slot for {} failed", contact_infos[0].id));
+        if current_slot > end_slot {
+            break;
+        }
+        if last_print.elapsed().as_secs() > 3 {
+            info!(
+                "{} current slot: {}, waiting for slot: {}",
+                test_name, current_slot, end_slot
+            );
+            last_print = Instant::now();
+        }
+    }
+
+    for (i, ingress_node) in contact_infos.iter().enumerate() {
+        let client = create_client(ingress_node.client_facing_addr(), VALIDATOR_PORT_RANGE);
+        assert_eq!(
+            client
+                .get_slot()
+                .unwrap_or_else(|_| panic!("get_slot for {} failed", ingress_node.id)),
+            roots[i]
+        );
+    }
+}
+
 fn poll_all_nodes_for_signature(
     entry_point_info: &ContactInfo,
     cluster_nodes: &[ContactInfo],

local-cluster/src/local_cluster.rs

@@ -360,6 +360,25 @@ impl LocalCluster {
         info!("{} done waiting for roots", test_name);
     }
 
+    pub fn check_no_new_roots(&self, num_slots_to_wait: usize, test_name: &str) {
+        let alive_node_contact_infos: Vec<_> = self
+            .validators
+            .values()
+            .map(|v| v.info.contact_info.clone())
+            .collect();
+        assert!(!alive_node_contact_infos.is_empty());
+        info!("{} discovering nodes", test_name);
+        let cluster_nodes = discover_cluster(
+            &alive_node_contact_infos[0].gossip,
+            alive_node_contact_infos.len(),
+        )
+        .unwrap();
+        info!("{} discovered {} nodes", test_name, cluster_nodes.len());
+        info!("{} making sure no new roots on any nodes", test_name);
+        cluster_tests::check_no_new_roots(num_slots_to_wait, &alive_node_contact_infos, test_name);
+        info!("{} done waiting for roots", test_name);
+    }
+
     fn transfer_with_client(
         client: &ThinClient,
         source_keypair: &Keypair,