net.sh: Refactor node initialization wait (#10819) (#10824)

* remote-node.sh: Factor out init wait to own script

* remote-node.sh: Allow nodes to initialize asynchronously

* testnet-automation: Plumb --async-node-init

(cherry picked from commit 7021e1c584)

Co-authored-by: Trent Nelson <trent@solana.com>

net/remote/remote-node-wait-init.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -e
+set -x
+
+initCompleteFile=init-complete-node.log
+waitTime=${1:=600}
+
+waitForNodeToInit() {
+  declare hostname
+  hostname=$(hostname)
+  echo "--- waiting for $hostname to boot up"
+  declare startTime=$SECONDS
+  while [[ ! -r $initCompleteFile ]]; do
+    declare timeWaited=$((SECONDS - startTime))
+    if [[ $timeWaited -ge $waitTime ]]; then
+      echo "^^^ +++"
+      echo "Error: $initCompleteFile not found in $timeWaited seconds"
+      exit 1
+    fi
+    echo "Waiting for $initCompleteFile ($timeWaited) on $hostname..."
+    sleep 5
+  done
+  echo "$hostname booted up"
+}
+
+cd ~/solana
+waitForNodeToInit