From 64ebd9a194a5375b6c401689890382021b719dd8 Mon Sep 17 00:00:00 2001
From: Michael Vines <mvines@gmail.com>
Date: Tue, 8 Jan 2019 14:29:24 -0800
Subject: [PATCH] Add update-to-restart operation.  Also try to update before
 restarting on sanity failures

---
 ci/testnet-manager.sh | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/ci/testnet-manager.sh b/ci/testnet-manager.sh
index 294421b609..9c72a4f3fb 100755
--- a/ci/testnet-manager.sh
+++ b/ci/testnet-manager.sh
@@ -50,6 +50,8 @@ steps:
             value: "sanity-or-restart"
           - label: "Start (or restart) the network"
             value: "start"
+          - label: "Update the network software.  Restart network on failure"
+            value: "update-or-restart"
           - label: "Stop the network"
             value: "stop"
           - label: "Sanity check only"
@@ -176,6 +178,7 @@ start() {
   else
     echo "--- stop $TESTNET"
   fi
+  declare maybeReuseLedger=$2
 
   case $TESTNET in
   testnet-edge)
@@ -186,6 +189,7 @@ start() {
       RUST_LOG=solana=info \
         ci/testnet-deploy.sh edge-testnet-solana-com ec2 us-west-1a \
           -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0ccd4f2239886fa94 \
+          ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-d}
     )
     ;;
@@ -197,6 +201,7 @@ start() {
         ci/testnet-deploy.sh edge-perf-testnet-solana-com ec2 us-west-2b \
           -g -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
+          ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-d}
     )
     ;;
@@ -209,6 +214,7 @@ start() {
         ci/testnet-deploy.sh beta-testnet-solana-com ec2 us-west-1a \
           -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0f286cf8a0771ce35 \
           -b \
+          ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-d}
     )
     ;;
@@ -220,6 +226,7 @@ start() {
         ci/testnet-deploy.sh beta-perf-testnet-solana-com ec2 us-west-2b \
           -g -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
+          ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-d}
     )
     ;;
@@ -232,9 +239,11 @@ start() {
         ci/testnet-deploy.sh testnet-solana-com ec2 us-west-1a \
           -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0fa502bf95f6f18b2 \
           -b \
+          ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-d}
         #ci/testnet-deploy.sh testnet-solana-com gce us-east1-c \
         #  -s "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a testnet-solana-com  \
+        #  ${maybeReuseLedger:+-r} \
         #  ${maybeDelete:+-d}
     )
     ;;
@@ -248,10 +257,12 @@ start() {
           -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
           -d pd-ssd \
+          ${maybeReuseLedger:+-r} \
           ${maybeDelete:+-d}
         #ci/testnet-deploy.sh perf-testnet-solana-com ec2 us-east-1a \
         #  -g \
         #  -t "$CHANNEL_OR_TAG" -c 2 \
+        #  ${maybeReuseLedger:+-r} \
         #  ${maybeDelete:+-d}
     )
     ;;
@@ -276,13 +287,28 @@ start)
 stop)
   stop
   ;;
+update-or-restart)
+  if start "" update; then
+    echo Update successful
+  else
+    echo "+++ Update failed, restarting the network"
+    $metricsWriteDatapoint "testnet-manager update-failure=1"
+    start
+  fi
+  ;;
 sanity-or-restart)
   if sanity; then
     echo Pass
   else
-    echo "+++ Sanity failed, restarting the network"
+    echo "+++ Sanity failed, updating the network"
     $metricsWriteDatapoint "testnet-manager sanity-failure=1"
-    start
+    if start "" update; then
+      echo Update successful
+    else
+      echo "+++ Update failed, restarting the network"
+      $metricsWriteDatapoint "testnet-manager update-failure=1"
+      start
+    fi
   fi
   ;;
 esac