Add validator startup process reporting before RPC is available

validator/src/dashboard.rs

@@ -5,6 +5,7 @@ use {
     solana_client::{
         client_error, rpc_client::RpcClient, rpc_request, rpc_response::RpcContactInfo,
     },
+    solana_core::validator::ValidatorStartProgress,
     solana_sdk::{
         clock::{Slot, DEFAULT_TICKS_PER_SLOT, MS_PER_TICK},
         commitment_config::CommitmentConfig,
@@ -13,13 +14,14 @@ use {
     },
     std::{
         io,
+        net::SocketAddr,
         path::{Path, PathBuf},
         sync::{
             atomic::{AtomicBool, Ordering},
             Arc,
         },
         thread,
-        time::Duration,
+        time::{Duration, SystemTime},
     },
 };
 
@@ -69,52 +71,26 @@ impl Dashboard {
         while !exit.load(Ordering::Relaxed) {
             let progress_bar = new_spinner_progress_bar();
             progress_bar.set_message("Connecting...");
-            let (start_time, rpc_client, identity) = loop {
-                if exit.load(Ordering::Relaxed) {
-                    return;
-                }
 
-                let admin_client = admin_rpc_service::connect(&ledger_path);
-                let (rpc_addr, start_time) = match runtime.block_on(async move {
-                    let admin_client = admin_client.await.map_err(|err| {
-                        format!("Unable to connect to validator process: {}", err)
-                    })?;
+            let (rpc_addr, start_time) = match runtime.block_on(wait_for_validator_startup(
+                &ledger_path,
+                &exit,
+                progress_bar,
+            )) {
+                None => continue,
+                Some(results) => results,
+            };
 
-                    let rpc_addr = admin_client
-                        .rpc_addr()
-                        .await
-                        .map_err(|err| format!("Unable to get validator RPC address: {}", err))?
-                        .ok_or_else(|| "RPC not available".to_string())?;
-
-                    let start_time = admin_client
-                        .start_time()
-                        .await
-                        .map_err(|err| format!("Unable to get validator start time: {}", err))?;
-
-                    Ok::<_, String>((rpc_addr, start_time))
-                }) {
-                    Ok((rpc_addr, start_time)) => (rpc_addr, start_time),
-                    Err(err) => {
-                        progress_bar.set_message(&format!("Connecting... ({})", err));
-                        thread::sleep(Duration::from_millis(500));
-                        continue;
-                    }
-                };
-
-                let rpc_client = RpcClient::new_socket(rpc_addr);
-
-                // Wait until RPC starts responding...
-                match rpc_client.get_identity() {
-                    Ok(identity) => break (start_time, rpc_client, identity),
-                    Err(err) => {
-                        progress_bar.set_message(&format!("Waiting for RPC... ({})", err));
-                    }
+            let rpc_client = RpcClient::new_socket(rpc_addr);
+            let identity = match rpc_client.get_identity() {
+                Ok(identity) => identity,
+                Err(err) => {
+                    println!("Failed to get validator identity over RPC: {}", err);
+                    continue;
                 }
             };
 
-            drop(progress_bar);
             println_name_value("Identity:", &identity.to_string());
-
             if let Some(contact_info) = get_contact_info(&rpc_client, &identity) {
                 println_name_value(
                     "Version:",
@@ -197,6 +173,61 @@ impl Dashboard {
     }
 }
 
+async fn wait_for_validator_startup(
+    ledger_path: &Path,
+    exit: &Arc<AtomicBool>,
+    progress_bar: ProgressBar,
+) -> Option<(SocketAddr, SystemTime)> {
+    let mut admin_client = None;
+    loop {
+        if exit.load(Ordering::Relaxed) {
+            return None;
+        }
+        thread::sleep(Duration::from_secs(1));
+
+        if admin_client.is_none() {
+            match admin_rpc_service::connect(&ledger_path).await {
+                Ok(new_admin_client) => admin_client = Some(new_admin_client),
+                Err(err) => {
+                    progress_bar.set_message(&format!("Unable to connect to validator: {}", err));
+                    continue;
+                }
+            }
+        }
+
+        match admin_client.as_ref().unwrap().start_progress().await {
+            Ok(start_progress) => {
+                if start_progress == ValidatorStartProgress::Running {
+                    let admin_client = admin_client.take().unwrap();
+
+                    match async move {
+                        let rpc_addr = admin_client.rpc_addr().await?;
+                        let start_time = admin_client.start_time().await?;
+                        Ok::<_, jsonrpc_core_client::RpcError>((rpc_addr, start_time))
+                    }
+                    .await
+                    {
+                        Ok((None, _)) => progress_bar.set_message(&"RPC service not available"),
+                        Ok((Some(rpc_addr), start_time)) => return Some((rpc_addr, start_time)),
+                        Err(err) => {
+                            progress_bar
+                                .set_message(&format!("Failed to get validator info: {}", err));
+                        }
+                    }
+                } else {
+                    progress_bar
+                        .set_message(&format!("Validator startup: {:?}...", start_progress));
+                }
+            }
+            Err(err) => {
+                admin_client = None;
+                progress_bar
+                    .set_message(&format!("Failed to get validator start progress: {}", err));
+            }
+        }
+    }
+}
+
 fn get_contact_info(rpc_client: &RpcClient, identity: &Pubkey) -> Option<RpcContactInfo> {
     rpc_client
         .get_cluster_nodes()