From e84f1f6de70ef1bb32b484b8c0c7154d233404e9 Mon Sep 17 00:00:00 2001
From: Michael Vines <mvines@gmail.com>
Date: Tue, 22 Jan 2019 11:39:10 -0800
Subject: [PATCH] Wait until the node successfully boots

---
 net/remote/remote-node.sh | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh
index c84aed714a..2d1965c2f5 100755
--- a/net/remote/remote-node.sh
+++ b/net/remote/remote-node.sh
@@ -148,7 +148,7 @@ local|tar)
     maybeNoLeaderRotation="--no-leader-rotation"
   fi
 
-
+  rm -f init-complete-file
   case $nodeType in
   bootstrap-leader)
     if [[ -e /dev/nvidia0 && -x ~/.cargo/bin/solana-fullnode-cuda ]]; then
@@ -160,7 +160,9 @@ local|tar)
       ./multinode-demo/setup.sh -t bootstrap-leader $setupArgs
     fi
     ./multinode-demo/drone.sh > drone.log 2>&1 &
-    ./multinode-demo/bootstrap-leader.sh $maybeNoLeaderRotation > bootstrap-leader.log 2>&1 &
+    ./multinode-demo/bootstrap-leader.sh \
+      --init-complete-file init-complete-file
+      $maybeNoLeaderRotation > bootstrap-leader.log 2>&1 &
     ln -sTf bootstrap-leader.log fullnode.log
     ;;
   fullnode)
@@ -175,13 +177,28 @@ local|tar)
     if [[ $skipSetup != true ]]; then
       ./multinode-demo/setup.sh -t fullnode $setupArgs
     fi
-    ./multinode-demo/fullnode.sh $maybeNoLeaderRotation "$entrypointIp":~/solana "$entrypointIp:8001" > fullnode.log 2>&1 &
+    ./multinode-demo/fullnode.sh \
+      --init-complete-file init-complete-file \
+      $maybeNoLeaderRotation \
+      "$entrypointIp":~/solana "$entrypointIp:8001" > fullnode.log 2>&1 &
     ;;
   *)
     echo "Error: unknown node type: $nodeType"
     exit 1
     ;;
   esac
+
+  SECONDS=0
+  while [[ ! -r init-complete-file ]]; do
+    if [[ $SECONDS -ge 30 ]]; then
+      echo "Error: node failed to boot in $SECONDS seconds"
+      exit 1
+    fi
+    echo "Waiting for node to boot ($SECONDS)..."
+    sleep 2
+  done
+  echo "Node booted in $SECONDS seconds"
+
   disown
   ;;
 *)