Checkout testnet scripts from tip to upstream/0.11 (#4016)

* Checkout testnet scripts from tip to upstream/0.11

* Add back NO_VALIDATOR_SANITY to testnet sanity

ci/testnet-deploy.sh

@@ -2,6 +2,7 @@
 set -e
 
 cd "$(dirname "$0")"/..
+source ci/upload-ci-artifact.sh
 
 zone=
 bootstrapFullNodeAddress=
@@ -9,13 +10,15 @@ bootstrapFullNodeMachineType=
 clientNodeCount=0
 additionalFullNodeCount=10
 publicNetwork=false
-snapChannel=edge
+skipSetup=false
+skipStart=false
+externalNode=false
 tarChannelOrTag=edge
 delete=false
 enableGpu=false
 bootDiskType=""
 leaderRotation=true
-useTarReleaseChannel=false
+blockstreamer=false
 
 usage() {
   exitcode=0
@@ -24,23 +27,23 @@ usage() {
     echo "Error: $*"
   fi
   cat <<EOF
-usage: $0 [name] [cloud] [zone] [options...]
+usage: $0 -p network-name -C cloud -z zone1 [-z zone2] ... [-z zoneN] [options...]
 
 Deploys a CD testnet
 
-  name  - name of the network
+  mandatory arguments:
-  cloud - cloud provider to use (gce, ec2)
+  -p [network-name]  - name of the network
-  zone  - cloud provider zone to deploy the network into
+  -C [cloud] - cloud provider to use (gce, ec2)
+  -z [zone]  - cloud provider zone to deploy the network into.  Must specify at least one zone
 
   options:
-   -s edge|beta|stable  - Deploy the specified Snap release channel
-                          (default: $snapChannel)
    -t edge|beta|stable|vX.Y.Z  - Deploy the latest tarball release for the
                                  specified release channel (edge|beta|stable) or release tag
                                  (vX.Y.Z)
                                  (default: $tarChannelOrTag)
    -n [number]          - Number of additional full nodes (default: $additionalFullNodeCount)
    -c [number]          - Number of client bencher nodes (default: $clientNodeCount)
+   -u                   - Include a Blockstreamer (default: $blockstreamer)
    -P                   - Use public network IP addresses (default: $publicNetwork)
    -G                   - Enable GPU, and set count/type of GPUs to use (e.g n1-standard-16 --accelerator count=4,type=nvidia-tesla-k80)
    -g                   - Enable GPU (default: $enableGpu)
@@ -48,6 +51,11 @@ Deploys a CD testnet
    -a [address]         - Set the bootstrap fullnode's external IP address to this GCE address
    -d [disk-type]       - Specify a boot disk type (default None) Use pd-ssd to get ssd on GCE.
    -D                   - Delete the network
+   -r                   - Reuse existing node/ledger configuration from a
+                          previous |start| (ie, don't run ./multinode-demo/setup.sh).
+   -x                   - External node.  Default: false
+   -s                   - Skip start.  Nodes will still be created or configured, but network software will not be started.
+   -S                   - Stop network software without tearing down nodes.
 
    Note: the SOLANA_METRICS_CONFIG environment variable is used to configure
          metrics
@@ -55,19 +63,22 @@ EOF
   exit $exitcode
 }
 
-netName=$1
+zone=()
-cloudProvider=$2
-zone=$3
-[[ -n $netName ]] || usage
-[[ -n $cloudProvider ]] || usage "Cloud provider not specified"
-[[ -n $zone ]] || usage "Zone not specified"
-shift 3
 
-while getopts "h?p:Pn:c:s:t:gG:a:Dbd:" opt; do
+while getopts "h?p:Pn:c:t:gG:a:Dbd:rusxz:p:C:S" opt; do
   case $opt in
   h | \?)
     usage
     ;;
+  p)
+    netName=$OPTARG
+    ;;
+  C)
+    cloudProvider=$OPTARG
+    ;;
+  z)
+    zone+=("$OPTARG")
+    ;;
   P)
     publicNetwork=true
     ;;
@@ -77,21 +88,10 @@ while getopts "h?p:Pn:c:s:t:gG:a:Dbd:" opt; do
   c)
     clientNodeCount=$OPTARG
     ;;
-  s)
-    case $OPTARG in
-    edge|beta|stable)
-      snapChannel=$OPTARG
-      ;;
-    *)
-      usage "Invalid snap channel: $OPTARG"
-      ;;
-    esac
-    ;;
   t)
     case $OPTARG in
     edge|beta|stable|v*)
       tarChannelOrTag=$OPTARG
-      useTarReleaseChannel=true
       ;;
     *)
       usage "Invalid release channel: $OPTARG"
@@ -117,53 +117,133 @@ while getopts "h?p:Pn:c:s:t:gG:a:Dbd:" opt; do
   D)
     delete=true
     ;;
+  r)
+    skipSetup=true
+    ;;
+  s)
+    skipStart=true
+    ;;
+  x)
+    externalNode=true
+    ;;
+  u)
+    blockstreamer=true
+    ;;
+  S)
+    stopNetwork=true
+    ;;
   *)
     usage "Error: unhandled option: $opt"
     ;;
   esac
 done
 
+[[ -n $netName ]] || usage
+[[ -n $cloudProvider ]] || usage "Cloud provider not specified"
+[[ -n ${zone[*]} ]] || usage "At least one zone must be specified"
 
-create_args=(
+shutdown() {
-  -a "$bootstrapFullNodeAddress"
+  exitcode=$?
-  -c "$clientNodeCount"
-  -n "$additionalFullNodeCount"
-  -p "$netName"
-  -z "$zone"
-)
 
-if [[ -n $bootDiskType ]]; then
+  set +e
-  create_args+=(-d "$bootDiskType")
+  if [[ -d net/log ]]; then
-fi
+    mv net/log net/log-deploy
-
+    for logfile in net/log-deploy/*; do
-if $enableGpu; then
+      if [[ -f $logfile ]]; then
-  if [[ -z $bootstrapFullNodeMachineType ]]; then
+        upload-ci-artifact "$logfile"
-    create_args+=(-g)
+        tail "$logfile"
-  else
+      fi
-    create_args+=(-G "$bootstrapFullNodeMachineType")
+    done
   fi
-fi
+  exit $exitcode
-
+}
-if ! $leaderRotation; then
+rm -rf net/{log,-deploy}
-  create_args+=(-b)
+trap shutdown EXIT INT
-fi
-
-if $publicNetwork; then
-  create_args+=(-P)
-fi
 
 set -x
 
-echo "--- $cloudProvider.sh delete"
+# Build a string to pass zone opts to $cloudProvider.sh: "-z zone1 -z zone2 ..."
-time net/"$cloudProvider".sh delete -z "$zone" -p "$netName"
+zone_args=()
-if $delete; then
+for val in "${zone[@]}"; do
-  exit 0
+  zone_args+=("-z $val")
+done
+
+if $stopNetwork; then
+  skipSetup=true
 fi
 
-echo "--- $cloudProvider.sh create"
+# Create the network
-time net/"$cloudProvider".sh create "${create_args[@]}"
+if ! $skipSetup; then
+  echo "--- $cloudProvider.sh delete"
+  # shellcheck disable=SC2068
+  time net/"$cloudProvider".sh delete ${zone_args[@]} -p "$netName" ${externalNode:+-x}
+  if $delete; then
+    exit 0
+  fi
+
+  echo "--- $cloudProvider.sh create"
+  create_args=(
+    -p "$netName"
+    -a "$bootstrapFullNodeAddress"
+    -c "$clientNodeCount"
+    -n "$additionalFullNodeCount"
+  )
+  # shellcheck disable=SC2206
+  create_args+=(${zone_args[@]})
+
+  if $blockstreamer; then
+    create_args+=(-u)
+  fi
+
+  if [[ -n $bootDiskType ]]; then
+    create_args+=(-d "$bootDiskType")
+  fi
+
+  if $enableGpu; then
+    if [[ -z $bootstrapFullNodeMachineType ]]; then
+      create_args+=(-g)
+    else
+      create_args+=(-G "$bootstrapFullNodeMachineType")
+    fi
+  fi
+
+  if ! $leaderRotation; then
+    create_args+=(-b)
+  fi
+
+  if $publicNetwork; then
+    create_args+=(-P)
+  fi
+
+  if $externalNode; then
+    create_args+=(-x)
+  fi
+
+  time net/"$cloudProvider".sh create "${create_args[@]}"
+else
+  echo "--- $cloudProvider.sh config"
+  config_args=(
+    -p "$netName"
+  )
+  # shellcheck disable=SC2206
+  config_args+=(${zone_args[@]})
+  if $publicNetwork; then
+    config_args+=(-P)
+  fi
+
+  time net/"$cloudProvider".sh config "${config_args[@]}"
+fi
 net/init-metrics.sh -e
 
+echo "+++ $cloudProvider.sh info"
+net/"$cloudProvider".sh info
+
+if $stopNetwork; then
+  echo --- net.sh stop
+  time net/net.sh stop
+  exit 0
+fi
+
 echo --- net.sh start
 maybeRejectExtraNodes=
 if ! $publicNetwork; then
@@ -177,10 +257,40 @@ maybeNoLedgerVerify=
 if [[ -n $NO_LEDGER_VERIFY ]]; then
   maybeNoLedgerVerify="-o noLedgerVerify"
 fi
-# shellcheck disable=SC2086 # Don't want to double quote maybeRejectExtraNodes
+
-if $useTarReleaseChannel; then
+maybeSkipSetup=
-  time net/net.sh start -t "$tarChannelOrTag" $maybeRejectExtraNodes $maybeNoValidatorSanity $maybeNoLedgerVerify
+if $skipSetup; then
-else
+  maybeSkipSetup="-r"
-  time net/net.sh start -s "$snapChannel" $maybeRejectExtraNodes $maybeNoValidatorSanity $maybeNoLedgerVerify
 fi
-exit 0
+
+ok=true
+if ! $skipStart; then
+  (
+    if $skipSetup; then
+      # TODO: Enable rolling updates
+      #op=update
+      op=restart
+    else
+      op=start
+    fi
+
+    maybeUpdateManifestKeypairFile=
+    # shellcheck disable=SC2154 # SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu comes from .buildkite/env/
+    if [[ -n $SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu ]]; then
+      echo "$SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu" > update_manifest_keypair.json
+      maybeUpdateManifestKeypairFile="-i update_manifest_keypair.json"
+    fi
+
+    # shellcheck disable=SC2086 # Don't want to double quote the $maybeXYZ variables
+    time net/net.sh $op -t "$tarChannelOrTag" \
+      $maybeUpdateManifestKeypairFile \
+      $maybeSkipSetup \
+      $maybeRejectExtraNodes \
+      $maybeNoValidatorSanity \
+      $maybeNoLedgerVerify
+  ) || ok=false
+
+  net/net.sh logs
+fi
+
+$ok

ci/testnet-manager.sh

@@ -42,18 +42,28 @@ steps:
             value: "testnet-beta"
           - label: "testnet-beta-perf"
             value: "testnet-beta-perf"
+          - label: "testnet-demo"
+            value: "testnet-demo"
       - select: "Operation"
         key: "testnet-operation"
         default: "sanity-or-restart"
         options:
-          - label: "Sanity check.  Restart network on failure"
+          - label: "Create new testnet nodes and then start network software.  If nodes are already created, they will be deleted and then re-created."
-            value: "sanity-or-restart"
+            value: "create-and-start"
-          - label: "Start (or restart) the network"
+          - label: "Create new testnet nodes, but do not start network software.  If nodes are already created, they will be deleted and then re-created."
+            value: "create"
+          - label: "Start network software on already-created testnet nodes.  If software is already running, it will be restarted."
             value: "start"
-          - label: "Stop the network"
+          - label: "Stop network software without deleting testnet nodes"
             value: "stop"
+          - label: "Update the network software.  Restart network software on failure"
+            value: "update-or-restart"
+          - label: "Sanity check.  Restart network software on failure"
+            value: "sanity-or-restart"
           - label: "Sanity check only"
             value: "sanity"
+          - label: "Delete all nodes on a testnet.  Network software will be stopped first if it is running"
+            value: "delete"
   - command: "ci/$(basename "$0")"
     agents:
       - "queue=$BUILDKITE_AGENT_META_DATA_QUEUE"
@@ -62,33 +72,51 @@ EOF
   exit 0
 fi
 
-export SOLANA_METRICS_CONFIG="db=$TESTNET,$SOLANA_METRICS_PARTIAL_CONFIG"
-echo "SOLANA_METRICS_CONFIG: $SOLANA_METRICS_CONFIG"
-
 ci/channel-info.sh
 eval "$(ci/channel-info.sh)"
 
+
+EC2_ZONES=(us-west-1a sa-east-1a ap-northeast-2a eu-central-1a ca-central-1a)
+GCE_ZONES=(us-west1-b asia-east2-a europe-west4-a southamerica-east1-b us-east4-c)
+case $TESTNET in
+testnet-edge|testnet-edge-perf)
+  CHANNEL_OR_TAG=edge
+  CHANNEL_BRANCH=$EDGE_CHANNEL
+  : "${TESTNET_DB_HOST:=https://clocktower-f1d56615.influxcloud.net:8086}"
+  ;;
+testnet-beta|testnet-beta-perf)
+  CHANNEL_OR_TAG=beta
+  CHANNEL_BRANCH=$BETA_CHANNEL
+  : "${TESTNET_DB_HOST:=https://clocktower-f1d56615.influxcloud.net:8086}"
+  : "${EC2_NODE_COUNT:=10}"
+  : "${GCE_NODE_COUNT:=}"
+  ;;
+testnet|testnet-perf)
+  CHANNEL_OR_TAG=$STABLE_CHANNEL_LATEST_TAG
+  CHANNEL_BRANCH=$STABLE_CHANNEL
+  : "${TESTNET_DB_HOST:=https://clocktower-f1d56615.influxcloud.net:8086}"
+  ;;
+testnet-demo)
+  CHANNEL_OR_TAG=beta
+  CHANNEL_BRANCH=$BETA_CHANNEL
+  ;;
+*)
+  echo "Error: Invalid TESTNET=$TESTNET"
+  exit 1
+  ;;
+esac
+
+if [[ -n $TESTNET_DB_HOST ]]; then
+  SOLANA_METRICS_PARTIAL_CONFIG="host=$TESTNET_DB_HOST,$SOLANA_METRICS_PARTIAL_CONFIG"
+fi
+
+export SOLANA_METRICS_CONFIG="db=$TESTNET,$SOLANA_METRICS_PARTIAL_CONFIG"
+echo "SOLANA_METRICS_CONFIG: $SOLANA_METRICS_CONFIG"
+source scripts/configure-metrics.sh
+
 if [[ -n $TESTNET_TAG ]]; then
   CHANNEL_OR_TAG=$TESTNET_TAG
 else
-  case $TESTNET in
-  testnet-edge|testnet-edge-perf)
-    CHANNEL_OR_TAG=edge
-    CHANNEL_BRANCH=$EDGE_CHANNEL
-    ;;
-  testnet-beta|testnet-beta-perf)
-    CHANNEL_OR_TAG=beta
-    CHANNEL_BRANCH=$BETA_CHANNEL
-    ;;
-  testnet|testnet-perf)
-    CHANNEL_OR_TAG=$STABLE_CHANNEL_LATEST_TAG
-    CHANNEL_BRANCH=$STABLE_CHANNEL
-    ;;
-  *)
-    echo "Error: Invalid TESTNET=$TESTNET"
-    exit 1
-    ;;
-  esac
 
   if [[ $BUILDKITE_BRANCH != "$CHANNEL_BRANCH" ]]; then
     (
@@ -102,79 +130,78 @@ steps:
       env:
         TESTNET: "$TESTNET"
         TESTNET_OP: "$TESTNET_OP"
+        TESTNET_DB_HOST: "$TESTNET_DB_HOST"
+        EC2_NODE_COUNT: "$EC2_NODE_COUNT"
+        GCE_NODE_COUNT: "$GCE_NODE_COUNT"
 EOF
     ) | buildkite-agent pipeline upload
     exit 0
   fi
 fi
 
-
 sanity() {
   echo "--- sanity $TESTNET"
   case $TESTNET in
   testnet-edge)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+        ci/testnet-sanity.sh edge-testnet-solana-com ec2 us-west-1a
-      ci/testnet-sanity.sh edge-testnet-solana-com ec2 us-west-1a
     )
     ;;
   testnet-edge-perf)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export REJECT_EXTRA_NODES=1
+      REJECT_EXTRA_NODES=1 \
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+      NO_VALIDATOR_SANITY=1 \
-      ci/testnet-sanity.sh edge-perf-testnet-solana-com ec2 us-west-2b
+        ci/testnet-sanity.sh edge-perf-testnet-solana-com ec2 us-west-2b
     )
     ;;
   testnet-beta)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+
-      export NO_VALIDATOR_SANITY=1
+      ok=true
-      ci/testnet-sanity.sh beta-testnet-solana-com ec2 us-west-1a
+      if [[ -n $EC2_NODE_COUNT ]]; then
+        NO_LEDGER_VERIFY=1 \
+          ci/testnet-sanity.sh beta-testnet-solana-com ec2 "${EC2_ZONES[0]}" || ok=false
+      elif [[ -n $GCE_NODE_COUNT ]]; then
+        NO_LEDGER_VERIFY=1 \
+          ci/testnet-sanity.sh beta-testnet-solana-com gce "${GCE_ZONES[0]}" || ok=false
+      else
+        echo "Error: no EC2 or GCE nodes"
+        ok=false
+      fi
+      $ok
     )
     ;;
   testnet-beta-perf)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export REJECT_EXTRA_NODES=1
+      REJECT_EXTRA_NODES=1 \
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+      NO_VALIDATOR_SANITY=1 \
-      ci/testnet-sanity.sh beta-perf-testnet-solana-com ec2 us-west-2b
+        ci/testnet-sanity.sh beta-perf-testnet-solana-com ec2 us-west-2b
     )
     ;;
   testnet)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+      NO_VALIDATOR_SANITY=1 \
+        ci/testnet-sanity.sh testnet-solana-com ec2 us-west-1a
       #ci/testnet-sanity.sh testnet-solana-com gce us-east1-c
-      ci/testnet-sanity.sh testnet-solana-com ec2 us-west-1a
     )
     ;;
   testnet-perf)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export REJECT_EXTRA_NODES=1
+      REJECT_EXTRA_NODES=1 \
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+      NO_VALIDATOR_SANITY=1 \
+        ci/testnet-sanity.sh perf-testnet-solana-com gce us-west1-b
       #ci/testnet-sanity.sh perf-testnet-solana-com ec2 us-east-1a
-      ci/testnet-sanity.sh perf-testnet-solana-com gce us-west1-b
     )
     ;;
   *)
@@ -184,100 +211,165 @@ sanity() {
   esac
 }
 
+deploy() {
+  declare maybeCreate=$1
+  declare maybeStart=$2
+  declare maybeStop=$3
+  declare maybeDelete=$4
 
-start() {
+  # Create or recreate the nodes
-  declare maybeDelete=$1
+  if [[ -z $maybeCreate ]]; then
-  if [[ -z $maybeDelete ]]; then
+    skipCreate=skip
-    echo "--- start $TESTNET"
   else
+    skipCreate=""
+    echo "--- create $TESTNET"
+  fi
+
+  # Start or restart the network software on the nodes
+  if [[ -z $maybeStart ]]; then
+    skipStart=skip
+  else
+    skipStart=""
+    echo "--- start $TESTNET"
+  fi
+
+  # Stop the nodes
+  if [[ -n $maybeStop ]]; then
     echo "--- stop $TESTNET"
   fi
 
+  # Delete the nodes
+  if [[ -n $maybeDelete ]]; then
+    echo "--- delete $TESTNET"
+  fi
+
   case $TESTNET in
   testnet-edge)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+      ci/testnet-deploy.sh -p edge-testnet-solana-com -C ec2 -z us-west-1a \
-      export NO_VALIDATOR_SANITY=1
+        -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P -a eipalloc-0ccd4f2239886fa94 \
-      ci/testnet-deploy.sh edge-testnet-solana-com ec2 us-west-1a \
+        ${skipCreate:+-r} \
-        -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0ccd4f2239886fa94 \
+        ${skipStart:+-s} \
+        ${maybeStop:+-S} \
         ${maybeDelete:+-D}
     )
     ;;
   testnet-edge-perf)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+      NO_VALIDATOR_SANITY=1 \
-      ci/testnet-deploy.sh edge-perf-testnet-solana-com ec2 us-west-2b \
+      RUST_LOG=solana=warn \
-        -g -t "$CHANNEL_OR_TAG" -c 2 \
+        ci/testnet-deploy.sh -p edge-perf-testnet-solana-com -C ec2 -z us-west-2b \
-        -b \
+          -g -t "$CHANNEL_OR_TAG" -c 2 \
-        ${maybeDelete:+-D}
+          -b \
+          ${skipCreate:+-r} \
+          ${skipStart:+-s} \
+          ${maybeStop:+-S} \
+          ${maybeDelete:+-D}
     )
     ;;
   testnet-beta)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+
-      export NO_VALIDATOR_SANITY=1
+      # Build an array to pass as opts to testnet-deploy.sh: "-z zone1 -z zone2 ..."
-      ci/testnet-deploy.sh beta-testnet-solana-com ec2 us-west-1a \
+      GCE_ZONE_ARGS=()
-        -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0f286cf8a0771ce35 \
+      for val in "${GCE_ZONES[@]}"; do
-        -b \
+        GCE_ZONE_ARGS+=("-z $val")
-        ${maybeDelete:+-D}
+      done
+
+      EC2_ZONE_ARGS=()
+      for val in "${EC2_ZONES[@]}"; do
+        EC2_ZONE_ARGS+=("-z $val")
+      done
+
+      if [[ -n $EC2_NODE_COUNT ]]; then
+        if [[ -n $GCE_NODE_COUNT ]] || [[ -n $skipStart ]]; then
+          maybeSkipStart="skip"
+        fi
+
+        # shellcheck disable=SC2068
+        ci/testnet-deploy.sh -p beta-testnet-solana-com -C ec2 ${EC2_ZONE_ARGS[@]} \
+          -t "$CHANNEL_OR_TAG" -n "$EC2_NODE_COUNT" -c 0 -u -P -a eipalloc-0f286cf8a0771ce35 \
+          ${skipCreate:+-r} \
+          ${maybeSkipStart:+-s} \
+          ${maybeStop:+-S} \
+          ${maybeDelete:+-D}
+      fi
+
+      if [[ -n $GCE_NODE_COUNT ]]; then
+        # shellcheck disable=SC2068
+        ci/testnet-deploy.sh -p beta-testnet-solana-com -C gce ${GCE_ZONE_ARGS[@]} \
+          -t "$CHANNEL_OR_TAG" -n "$GCE_NODE_COUNT" -c 0 -P \
+          ${skipCreate:+-r} \
+          ${skipStart:+-s} \
+          ${maybeStop:+-S} \
+          ${maybeDelete:+-D} \
+          ${EC2_NODE_COUNT:+-x}
+      fi
     )
     ;;
   testnet-beta-perf)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+      NO_VALIDATOR_SANITY=1 \
-      ci/testnet-deploy.sh beta-perf-testnet-solana-com ec2 us-west-2b \
+      RUST_LOG=solana=warn \
-        -g -t "$CHANNEL_OR_TAG" -c 2 \
+        ci/testnet-deploy.sh -p beta-perf-testnet-solana-com -C ec2 -z us-west-2b \
-        -b \
+          -g -t "$CHANNEL_OR_TAG" -c 2 \
-        ${maybeDelete:+-D}
+          -b \
+          ${skipCreate:+-r} \
+          ${skipStart:+-s} \
+          ${maybeStop:+-S} \
+          ${maybeDelete:+-D}
     )
     ;;
   testnet)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+      NO_VALIDATOR_SANITY=1 \
-      export NO_VALIDATOR_SANITY=1
+        ci/testnet-deploy.sh -p testnet-solana-com -C ec2 -z us-west-1a \
-      #ci/testnet-deploy.sh testnet-solana-com gce us-east1-c \
+          -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P -a eipalloc-0fa502bf95f6f18b2 \
-      #  -s "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a testnet-solana-com  \
+          -b \
-      #  ${maybeDelete:+-D}
+          ${skipCreate:+-r} \
-      ci/testnet-deploy.sh testnet-solana-com ec2 us-west-1a \
+          ${skipStart:+-s} \
-        -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0fa502bf95f6f18b2 \
+          ${maybeStop:+-S} \
-        -b \
+          ${maybeDelete:+-D}
-        ${maybeDelete:+-D}
+        #ci/testnet-deploy.sh -p testnet-solana-com -C gce -z us-east1-c \
+        #  -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a testnet-solana-com  \
+        #  ${maybeReuseLedger:+-r} \
+        #  ${maybeDelete:+-D}
     )
     ;;
   testnet-perf)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+      NO_VALIDATOR_SANITY=1 \
-      ci/testnet-deploy.sh perf-testnet-solana-com gce us-west1-b \
+      RUST_LOG=solana=warn \
-        -G "n1-standard-16 --accelerator count=2,type=nvidia-tesla-v100" \
+        ci/testnet-deploy.sh -p perf-testnet-solana-com -C gce -z us-west1-b \
-        -t "$CHANNEL_OR_TAG" -c 2 \
+          -G "--machine-type n1-standard-16 --accelerator count=2,type=nvidia-tesla-v100" \
-        -b \
+          -t "$CHANNEL_OR_TAG" -c 2 \
-        -d pd-ssd \
+          -b \
-        ${maybeDelete:+-D}
+          -d pd-ssd \
-      #ci/testnet-deploy.sh perf-testnet-solana-com ec2 us-east-1a \
+          ${skipCreate:+-r} \
-      #  -g \
+          ${skipStart:+-s} \
-      #  -t "$CHANNEL_OR_TAG" -c 2 \
+          ${maybeStop:+-S} \
-      #  ${maybeDelete:+-D}
+          ${maybeDelete:+-D}
+        #ci/testnet-deploy.sh -p perf-testnet-solana-com -C ec2 -z us-east-1a \
+        #  -g \
+        #  -t "$CHANNEL_OR_TAG" -c 2 \
+        #  ${maybeReuseLedger:+-r} \
+        #  ${maybeDelete:+-D}
+    )
+    ;;
+  testnet-demo)
+    (
+      set -x
+      echo "Demo net not yet implemented!"
+      exit 1
     )
     ;;
   *)
@@ -287,29 +379,117 @@ start() {
   esac
 }
 
+ENABLED_LOCKFILE="${HOME}/${TESTNET}.is_enabled"
+CREATED_LOCKFILE="${HOME}/${TESTNET}.is_created"
+
+create-and-start() {
+  rm -f "${CREATED_LOCKFILE}"
+  deploy create start
+  touch "${CREATED_LOCKFILE}"
+}
+create() {
+  rm -f "${CREATED_LOCKFILE}"
+  deploy create
+  touch "${CREATED_LOCKFILE}"
+}
+start() {
+  if [[ -f ${CREATED_LOCKFILE} ]]; then
+    deploy "" start
+  else
+    echo "Unable to start ${TESTNET}.  Are the nodes created?
+    Re-run ci/testnet-manager.sh with \$TESTNET_OP=create or \$TESTNET_OP=create-and-start"
+    exit 1
+  fi
+}
 stop() {
-  start delete
+  deploy "" ""
+}
+delete() {
+  deploy "" "" "" delete
+  rm -f "${CREATED_LOCKFILE}"
+}
+enable_testnet() {
+  touch "${ENABLED_LOCKFILE}"
+}
+disable_testnet() {
+  rm -f "${ENABLED_LOCKFILE}"
+}
+is_testnet_enabled() {
+  if [[ ! -f ${ENABLED_LOCKFILE} ]]; then
+    echo "--- ${TESTNET} is currently disabled.  Enable ${TESTNET} by running ci/testnet-manager.sh with \$TESTNET_OP=enable, then re-run with current settings."
+    exit 0
+  fi
 }
 
 case $TESTNET_OP in
-sanity)
+enable)
-  sanity
+  enable_testnet
+  ;;
+disable)
+  delete
+  disable_testnet
+  ;;
+create-and-start)
+  is_testnet_enabled
+  create-and-start
+  ;;
+create)
+  is_testnet_enabled
+  create
   ;;
 start)
+  is_testnet_enabled
   start
   ;;
 stop)
+  is_testnet_enabled
   stop
   ;;
+sanity)
+  is_testnet_enabled
+  sanity
+  ;;
+delete)
+  is_testnet_enabled
+  delete
+  ;;
+update-or-restart)
+  is_testnet_enabled
+  if start; then
+    echo Update successful
+  else
+    echo "+++ Update failed, restarting the network"
+    $metricsWriteDatapoint "testnet-manager update-failure=1"
+    create-and-start
+  fi
+  ;;
 sanity-or-restart)
+  is_testnet_enabled
   if sanity; then
     echo Pass
   else
-    echo "Sanity failed, restarting the network"
+    echo "+++ Sanity failed, updating the network"
-    echo "^^^ +++"
+    $metricsWriteDatapoint "testnet-manager sanity-failure=1"
-    start
+
+    # TODO: Restore attempt to restart the cluster before recreating it
+    #       See https://github.com/solana-labs/solana/issues/3774
+    if false; then
+      if start; then
+        echo Update successful
+      else
+        echo "+++ Update failed, restarting the network"
+        $metricsWriteDatapoint "testnet-manager update-failure=1"
+        create-and-start
+      fi
+    else
+      create-and-start
+    fi
   fi
   ;;
+*)
+  echo "Error: Invalid TESTNET_OP=$TESTNET_OP"
+  exit 1
+  ;;
 esac
 
 echo --- fin