Refactored large network test to use finality to assert success (#978)

tests/multinode.rs

@@ -17,9 +17,8 @@ use solana::result;
 use solana::service::Service;
 use solana::signature::{Keypair, KeypairUtil, Pubkey};
 use solana::thin_client::ThinClient;
-use solana::timing::duration_as_s;
+use solana::timing::{duration_as_ms, duration_as_s};
 use solana::window::{default_window, WINDOW_SIZE};
-use std::cmp::max;
 use std::env;
 use std::fs::{copy, create_dir_all, remove_dir_all};
 use std::net::UdpSocket;
@@ -524,15 +523,7 @@ fn test_multi_node_dynamic_network() {
         Ok(val) => val
             .parse()
             .expect(&format!("env var {} is not parse-able as usize", key)),
-        Err(_) => 170,
+        Err(_) => 200,
-    };
-
-    let purge_key = "SOLANA_DYNAMIC_NODES_PURGE_LAG";
-    let purge_lag: usize = match env::var(purge_key) {
-        Ok(val) => val
-            .parse()
-            .expect(&format!("env var {} is not parse-able as usize", purge_key)),
-        Err(_) => std::usize::MAX,
     };
 
     let leader_keypair = Keypair::new();
@@ -631,94 +622,78 @@ fn test_multi_node_dynamic_network() {
 
     let mut validators: Vec<_> = t2.into_iter().map(|t| t.join().unwrap()).collect();
 
-    let now = Instant::now();
+    let mut client = mk_client(&leader_data);
+    let mut last_finality = client.get_finality();
+    info!("Last finality {}", last_finality);
+    let start = Instant::now();
     let mut consecutive_success = 0;
-    let mut failures = 0;
+    let mut expected_balance = leader_balance;
-    let mut max_distance_increase = 0i64;
     for i in 0..std::cmp::max(20, num_nodes) {
-        //verify leader can do transfer
+        trace!("getting leader last_id");
-        let expected = ((i + 3) * 500) as i64;
+        let last_id = client.get_last_id();
-        let leader_balance = retry_send_tx_and_retry_get_balance(
+        trace!("executing leader transfer");
-            &leader_data,
+        let sig = client
-            &alice_arc.read().unwrap(),
+            .transfer(
-            &bob_pubkey,
+                500,
-            Some(expected),
+                &alice_arc.read().unwrap().keypair(),
-        ).unwrap();
+                bob_pubkey,
-        if leader_balance != expected {
+                &last_id,
-            info!(
+            )
-                "leader dropped transaction {} {:?} {:?}",
+            .unwrap();
-                i, leader_balance, expected
+
-            );
+        expected_balance += 500;
+
+        assert!(client.poll_for_signature(&sig).is_ok());
+
+        let now = Instant::now();
+        let mut finality = client.get_finality();
+
+        // Need this to make sure the finality is updated
+        // (i.e. the node is not returning stale value)
+        while last_finality == finality {
+            finality = client.get_finality();
         }
-        //verify all validators have the same balance
+
-        {
+        while duration_as_ms(&now.elapsed()) < finality as u64 {
-            let mut success = 0usize;
+            sleep(Duration::from_millis(100));
-            let mut max_distance = 0i64;
+            finality = client.get_finality()
-            let mut total_distance = 0i64;
+        }
+
+        last_finality = finality;
+
+        let balance = retry_get_balance(&mut client, &bob_pubkey, Some(expected_balance));
+        assert_eq!(balance, Some(expected_balance));
+        consecutive_success += 1;
+
+        info!(
+            "SUCCESS[{}] balance: {}, finality: {} ms",
+            i, expected_balance, last_finality,
+        );
+
+        if consecutive_success == 10 {
+            info!("Took {} s to converge", duration_as_s(&start.elapsed()),);
+            info!("Verifying signature of the last transaction in the validators");
+
             let mut num_nodes_behind = 0i64;
             validators.retain(|server| {
-                let mut retain_me = true;
                 let mut client = mk_client(&server.0);
-                trace!("{:x} {} get_balance start", server.0.debug_id(), i);
+                trace!("{:x} polling for signature", server.0.debug_id());
-                let getbal = retry_get_balance(&mut client, &bob_pubkey, Some(leader_balance));
+                num_nodes_behind += match client.poll_for_signature(&sig) {
-                trace!(
+                    Ok(_) => 0,
-                    "{:x} {} get_balance: {:?} leader_balance: {}",
+                    Err(_) => 1,
-                    server.0.debug_id(),
+                };
-                    i,
+                true
-                    getbal,
-                    leader_balance
-                );
-                let bal = getbal.unwrap_or(0);
-                let distance = (leader_balance - bal) / 500;
-                max_distance = max(distance, max_distance);
-                total_distance += distance;
-                if distance > max_distance_increase {
-                    info!("Node {:x} is behind by {}", server.0.debug_id(), distance);
-                    max_distance_increase = distance;
-                    if max_distance_increase as u64 > purge_lag as u64 {
-                        server.1.exit();
-                        info!("Node {:x} is exiting", server.0.debug_id());
-                        retain_me = false;
-                    }
-                }
-                if distance > 0 {
-                    num_nodes_behind += 1;
-                }
-                if let Some(bal) = getbal {
-                    if bal == leader_balance {
-                        success += 1;
-                    }
-                }
-                retain_me
             });
-            if num_nodes_behind != 0 {
+
-                info!("{} nodes are lagging behind leader", num_nodes_behind);
-            }
             info!(
-                "SUCCESS[{}] {} out of {} distance: {} max_distance: {}  finality: {}",
+                "Validators lagging: {}/{}",
-                i,
+                num_nodes_behind,
-                success,
                 validators.len(),
-                total_distance,
-                max_distance,
-                get_finality(&leader_data)
             );
-            if success == validators.len() && total_distance == 0 {
+            break;
-                consecutive_success += 1;
-            } else {
-                consecutive_success = 0;
-                failures += 1;
-            }
-            if consecutive_success == 10 {
-                break;
-            }
         }
     }
-    info!(
+
-        "Took {} s to converge total failures: {}",
-        duration_as_s(&now.elapsed()),
-        failures
-    );
     assert_eq!(consecutive_success, 10);
     for (_, node) in &validators {
         node.exit();
@@ -821,9 +796,3 @@ fn retry_send_tx_and_retry_get_balance(
     }
     None
 }
-
-fn get_finality(leader: &NodeInfo) -> usize {
-    let mut client = mk_client(leader);
-    trace!("getting leader finality");
-    client.get_finality()
-}