net.sh: Refactor node initialization wait (#10819) (#10824)

* remote-node.sh: Factor out init wait to own script

* remote-node.sh: Allow nodes to initialize asynchronously

* testnet-automation: Plumb --async-node-init

(cherry picked from commit 7021e1c584)

Co-authored-by: Trent Nelson <trent@solana.com>

net/net.sh

@@ -277,6 +277,7 @@ startBootstrapLeader() {
          \"$gpuMode\" \
          \"$GEOLOCATION_API_KEY\" \
          \"$maybeWarpSlot\" \
+         \"$waitForNodeInit\" \
       "
 
   ) >> "$logFile" 2>&1 || {
@@ -347,6 +348,7 @@ startNode() {
          \"$gpuMode\" \
          \"$GEOLOCATION_API_KEY\" \
          \"$maybeWarpSlot\" \
+         \"$waitForNodeInit\" \
       "
   ) >> "$logFile" 2>&1 &
   declare pid=$!
@@ -582,6 +584,19 @@ deploy() {
     fi
   done
 
+  if ! $waitForNodeInit; then
+    # Handle async init
+    declare startTime=$SECONDS
+    for ipAddress in "${validatorIpList[@]}" "${blockstreamerIpList[@]}"; do
+      declare timeWaited=$((SECONDS - startTime))
+      if [[ $timeWaited -gt 600 ]]; then
+        break
+      fi
+      ssh "${sshOptions[@]}" -n "$ipAddress" \
+        "./solana/net/remote/remote-node-wait-init.sh $((600 - timeWaited))"
+    done
+  fi
+
   $metricsWriteDatapoint "testnet-deploy net-validators-started=1"
   additionalNodeDeployTime=$SECONDS
 
@@ -739,6 +754,7 @@ netemCommand="add"
 clientDelayStart=0
 netLogDir=
 maybeWarpSlot=
+waitForNodeInit=true
 
 command=$1
 [[ -n $command ]] || usage
@@ -845,6 +861,9 @@ while [[ -n $1 ]]; do
     elif [[ $1 == --warp-slot ]]; then
       maybeWarpSlot="$1 $2"
       shift 2
+    elif [[ $1 == --async-node-init ]]; then
+      waitForNodeInit=false
+      shift 1
     else
       usage "Unknown long option: $1"
     fi