Async poh verify (#6353)

* Async poh verify

* Up ticks_per_s to 160

GPU poh verify needs shorter poh sequences or it takes forever to
verify. Keep slot time the same at 400ms.

* Fix stats

* Don't halt on ticks

* Increase retries for local_cluster tests and make repairman test serial

local_cluster/src/cluster_tests.rs

@@ -57,7 +57,7 @@ pub fn spend_and_verify_all_nodes<S: ::std::hash::BuildHasher>(
             system_transaction::transfer(&funding_keypair, &random_keypair.pubkey(), 1, blockhash);
         let confs = VOTE_THRESHOLD_DEPTH + 1;
         let sig = client
-            .retry_transfer_until_confirmed(&funding_keypair, &mut transaction, 5, confs)
+            .retry_transfer_until_confirmed(&funding_keypair, &mut transaction, 10, confs)
             .unwrap();
         for validator in &cluster_nodes {
             if ignore_nodes.contains(&validator.id) {

local_cluster/src/local_cluster.rs

@@ -434,7 +434,7 @@ impl LocalCluster {
             *dest_pubkey
         );
         client
-            .retry_transfer(&source_keypair, &mut tx, 5)
+            .retry_transfer(&source_keypair, &mut tx, 10)
             .expect("client transfer");
         client
             .wait_for_balance(dest_pubkey, Some(lamports))
@@ -472,7 +472,7 @@ impl LocalCluster {
                 client.get_recent_blockhash().unwrap().0,
             );
             client
-                .retry_transfer(&from_account, &mut transaction, 5)
+                .retry_transfer(&from_account, &mut transaction, 10)
                 .expect("fund vote");
             client
                 .wait_for_balance(&vote_account_pubkey, Some(amount))
@@ -571,7 +571,7 @@ impl LocalCluster {
         let blockhash = client.get_recent_blockhash().unwrap().0;
         let mut transaction = Transaction::new(&signer_keys, message, blockhash);
         client
-            .retry_transfer(&from_keypair, &mut transaction, 5)
+            .retry_transfer(&from_keypair, &mut transaction, 10)
             .map(|_signature| ())
     }
 }

local_cluster/src/tests/local_cluster.rs

@@ -624,6 +624,7 @@ fn test_faulty_node(faulty_node_type: BroadcastStageType) {
 }
 
 #[test]
+#[serial]
 fn test_repairman_catchup() {
     solana_logger::setup();
     error!("test_repairman_catchup");